2024-04-05T02:31:11.873 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-05T02:31:11.873 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --verbose --archive /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230 --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --name teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi -- /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/orig.config.yaml 2024-04-05T02:31:11.939 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-05T02:31:12.053 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' first_in_suite: false job_id: '7641230' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: 9f192350ba5fe6d0a365b251a8c2af61f56a5fa6 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 8639 sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 27/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 9f192350ba5fe6d0a365b251a8c2af61f56a5fa6 targets: smithi150.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMIuUwNr2r/Tbm93vuIaJO2JQe+gvaskSlRuWjiq8XpYbdkqX6eS/NfxknSXO39TxGsSTrzk5nRpajplQ0nP90Y= smithi196.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO3Or61tPDasmByHPKt6guIqEGtWzShtMvTuJPVqqDYiej8lBCagYKGCJwl0jbWUkfeX9SVuJTH+vH9/kpATp+w= tasks: - pexec: all: - sudo dnf install runc -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-05_02:08:13 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-05T02:31:12.053 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa; will attempt to use it 2024-04-05T02:31:12.054 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks 2024-04-05T02:31:12.054 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-05T02:31:12.056 INFO:teuthology.task.internal:Checking packages... 2024-04-05T02:31:12.079 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '0028525c15e19309c20289c609e4fb9d0b12dd06' 2024-04-05T02:31:12.079 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:31:12.079 INFO:teuthology.packaging:ref: None 2024-04-05T02:31:12.079 INFO:teuthology.packaging:tag: None 2024-04-05T02:31:12.079 INFO:teuthology.packaging:branch: squid 2024-04-05T02:31:12.079 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:31:12.079 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-05T02:31:12.383 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-2013.g0028525c 2024-04-05T02:31:12.385 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-05T02:31:12.391 INFO:teuthology.task.internal:no buildpackages task found 2024-04-05T02:31:12.391 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-05T02:31:12.434 INFO:teuthology.task.internal:Saving configuration 2024-04-05T02:31:12.458 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-05T02:31:12.464 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-05T02:31:12.487 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:25:58.551901', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMIuUwNr2r/Tbm93vuIaJO2JQe+gvaskSlRuWjiq8XpYbdkqX6eS/NfxknSXO39TxGsSTrzk5nRpajplQ0nP90Y='} 2024-04-05T02:31:12.512 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi196.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:25:58.549985', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO3Or61tPDasmByHPKt6guIqEGtWzShtMvTuJPVqqDYiej8lBCagYKGCJwl0jbWUkfeX9SVuJTH+vH9/kpATp+w='} 2024-04-05T02:31:12.512 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-05T02:31:12.518 INFO:teuthology.task.internal:roles: ubuntu@smithi150.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-05T02:31:12.518 INFO:teuthology.task.internal:roles: ubuntu@smithi196.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-05T02:31:12.518 INFO:teuthology.run_tasks:Running task console_log... 2024-04-05T02:31:12.617 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f4669a190d0>, signals=[15]) 2024-04-05T02:31:12.617 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-05T02:31:12.625 INFO:teuthology.task.internal:Opening connections... 2024-04-05T02:31:12.625 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi150.front.sepia.ceph.com 2024-04-05T02:31:12.628 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:31:12.720 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi196.front.sepia.ceph.com 2024-04-05T02:31:12.721 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:31:12.813 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-05T02:31:12.820 DEBUG:teuthology.orchestra.run.smithi150:> uname -m 2024-04-05T02:31:12.837 INFO:teuthology.orchestra.run.smithi150.stdout:x86_64 2024-04-05T02:31:12.838 DEBUG:teuthology.orchestra.run.smithi150:> cat /etc/os-release 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:NAME="CentOS Stream" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION="9" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:ID="centos" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:ID_LIKE="rhel fedora" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:VERSION_ID="9" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:PLATFORM_ID="platform:el9" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:ANSI_COLOR="0;31" 2024-04-05T02:31:12.895 INFO:teuthology.orchestra.run.smithi150.stdout:LOGO="fedora-logo-icon" 2024-04-05T02:31:12.896 INFO:teuthology.orchestra.run.smithi150.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-05T02:31:12.896 INFO:teuthology.orchestra.run.smithi150.stdout:HOME_URL="https://centos.org/" 2024-04-05T02:31:12.896 INFO:teuthology.orchestra.run.smithi150.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-05T02:31:12.896 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-05T02:31:12.896 INFO:teuthology.orchestra.run.smithi150.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-05T02:31:12.897 INFO:teuthology.lock.ops:Updating smithi150.front.sepia.ceph.com on lock server 2024-04-05T02:31:12.921 DEBUG:teuthology.orchestra.run.smithi196:> uname -m 2024-04-05T02:31:12.938 INFO:teuthology.orchestra.run.smithi196.stdout:x86_64 2024-04-05T02:31:12.938 DEBUG:teuthology.orchestra.run.smithi196:> cat /etc/os-release 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:NAME="CentOS Stream" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:VERSION="9" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:ID="centos" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:ID_LIKE="rhel fedora" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:VERSION_ID="9" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:PLATFORM_ID="platform:el9" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:ANSI_COLOR="0;31" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:LOGO="fedora-logo-icon" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-05T02:31:12.995 INFO:teuthology.orchestra.run.smithi196.stdout:HOME_URL="https://centos.org/" 2024-04-05T02:31:12.996 INFO:teuthology.orchestra.run.smithi196.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-05T02:31:12.996 INFO:teuthology.orchestra.run.smithi196.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-05T02:31:12.996 INFO:teuthology.orchestra.run.smithi196.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-05T02:31:12.996 INFO:teuthology.lock.ops:Updating smithi196.front.sepia.ceph.com on lock server 2024-04-05T02:31:13.012 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-05T02:31:13.023 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-05T02:31:13.043 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-05T02:31:13.043 DEBUG:teuthology.orchestra.run.smithi150:> test '!' -e /home/ubuntu/cephtest 2024-04-05T02:31:13.046 DEBUG:teuthology.orchestra.run.smithi196:> test '!' -e /home/ubuntu/cephtest 2024-04-05T02:31:13.062 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-05T02:31:13.067 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-05T02:31:13.067 DEBUG:teuthology.orchestra.run.smithi150:> test -z $(ls -A /var/lib/ceph) 2024-04-05T02:31:13.102 DEBUG:teuthology.orchestra.run.smithi196:> test -z $(ls -A /var/lib/ceph) 2024-04-05T02:31:13.210 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-05T02:31:13.257 INFO:teuthology.run_tasks:Running task kernel... 2024-04-05T02:31:13.275 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-05T02:31:13.275 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-05T02:31:13.275 DEBUG:teuthology.orchestra.run.smithi150:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-05T02:31:13.276 DEBUG:teuthology.orchestra.run.smithi196:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-05T02:31:13.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:31:13.294 DEBUG:teuthology.orchestra.run.smithi196:> uname -r 2024-04-05T02:31:13.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:31:13.296 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2024-04-05T02:31:13.351 INFO:teuthology.orchestra.run.smithi196.stdout:5.14.0-361.el9.x86_64 2024-04-05T02:31:13.351 INFO:teuthology.task.kernel:Running kernel on smithi196: 5.14.0-361.el9.x86_64 2024-04-05T02:31:13.351 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum install -y kernel 2024-04-05T02:31:13.353 INFO:teuthology.orchestra.run.smithi150.stdout:5.14.0-361.el9.x86_64 2024-04-05T02:31:13.353 INFO:teuthology.task.kernel:Running kernel on smithi150: 5.14.0-361.el9.x86_64 2024-04-05T02:31:13.354 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum install -y kernel 2024-04-05T02:31:15.449 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-04-05T02:31:15.463 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-04-05T02:31:17.074 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS-9 - BaseOS 3.8 MB/s | 2.0 MB 00:00 2024-04-05T02:31:17.105 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS-9 - BaseOS 3.8 MB/s | 2.0 MB 00:00 2024-04-05T02:31:18.023 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.0 MB 00:00 2024-04-05T02:31:18.324 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - BaseOS 9.5 MB/s | 8.0 MB 00:00 2024-04-05T02:31:19.790 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - AppStream 27 MB/s | 19 MB 00:00 2024-04-05T02:31:24.043 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - CRB 8.1 MB/s | 6.1 MB 00:00 2024-04-05T02:31:25.500 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - Extras packages 50 kB/s | 16 kB 00:00 2024-04-05T02:31:28.421 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - AppStream 2.2 MB/s | 19 MB 00:08 2024-04-05T02:31:28.484 INFO:teuthology.orchestra.run.smithi196.stdout:Extra Packages for Enterprise Linux 7.3 MB/s | 21 MB 00:02 2024-04-05T02:31:32.805 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - CRB 6.8 MB/s | 6.1 MB 00:00 2024-04-05T02:31:33.492 INFO:teuthology.orchestra.run.smithi196.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-04-05T02:31:34.665 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - Extras packages 22 kB/s | 16 kB 00:00 2024-04-05T02:31:35.774 INFO:teuthology.orchestra.run.smithi196.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-05T02:31:35.774 INFO:teuthology.orchestra.run.smithi196.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-05T02:31:35.868 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout: kernel x86_64 5.14.0-435.el9 CentOS-BaseOS 212 k 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-core x86_64 5.14.0-435.el9 CentOS-BaseOS 15 M 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-modules x86_64 5.14.0-435.el9 CentOS-BaseOS 34 M 2024-04-05T02:31:35.878 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-modules-core x86_64 5.14.0-435.el9 CentOS-BaseOS 28 M 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout:Install 4 Packages 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 77 M 2024-04-05T02:31:35.879 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 123 M 2024-04-05T02:31:35.880 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:31:36.035 INFO:teuthology.orchestra.run.smithi196.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.3 MB/s | 212 kB 00:00 2024-04-05T02:31:36.719 INFO:teuthology.orchestra.run.smithi196.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 18 MB/s | 15 MB 00:00 2024-04-05T02:31:36.757 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 11 MB/s | 21 MB 00:01 2024-04-05T02:31:37.119 INFO:teuthology.orchestra.run.smithi196.stdout:(3/4): kernel-modules-core-5.14.0-435.el9.x86_6 26 MB/s | 28 MB 00:01 2024-04-05T02:31:37.494 INFO:teuthology.orchestra.run.smithi196.stdout:(4/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 21 MB/s | 34 MB 00:01 2024-04-05T02:31:37.497 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:31:37.497 INFO:teuthology.orchestra.run.smithi196.stdout:Total 47 MB/s | 77 MB 00:01 2024-04-05T02:31:37.497 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:31:37.853 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:31:37.853 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:31:38.715 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:31:38.716 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:31:41.014 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:31:41.586 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:31:41.693 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:31:41.940 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 5.2 kB/s | 1.7 kB 00:00 2024-04-05T02:31:44.197 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-05T02:31:44.198 INFO:teuthology.orchestra.run.smithi150.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-05T02:31:44.292 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout: kernel x86_64 5.14.0-435.el9 CentOS-BaseOS 212 k 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-core x86_64 5.14.0-435.el9 CentOS-BaseOS 15 M 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules x86_64 5.14.0-435.el9 CentOS-BaseOS 34 M 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules-core x86_64 5.14.0-435.el9 CentOS-BaseOS 28 M 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:31:44.299 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:31:44.300 INFO:teuthology.orchestra.run.smithi150.stdout:Install 4 Packages 2024-04-05T02:31:44.300 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:31:44.300 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 77 M 2024-04-05T02:31:44.300 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 123 M 2024-04-05T02:31:44.300 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:31:44.458 INFO:teuthology.orchestra.run.smithi150.stdout:(1/4): kernel-5.14.0-435.el9.x86_64.rpm 1.3 MB/s | 212 kB 00:00 2024-04-05T02:31:44.473 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:31:44.604 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:31:44.942 INFO:teuthology.orchestra.run.smithi150.stdout:(2/4): kernel-core-5.14.0-435.el9.x86_64.rpm 24 MB/s | 15 MB 00:00 2024-04-05T02:31:45.551 INFO:teuthology.orchestra.run.smithi150.stdout:(3/4): kernel-modules-5.14.0-435.el9.x86_64.rpm 27 MB/s | 34 MB 00:01 2024-04-05T02:31:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:(4/4): kernel-modules-core-5.14.0-435.el9.x86_6 20 MB/s | 28 MB 00:01 2024-04-05T02:31:45.862 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:31:45.862 INFO:teuthology.orchestra.run.smithi150.stdout:Total 49 MB/s | 77 MB 00:01 2024-04-05T02:31:45.863 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:31:46.236 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:31:46.237 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:31:47.105 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:31:47.105 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:31:49.319 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:31:49.454 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:31:49.536 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:31:50.113 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-modules-core-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:31:50.202 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:31:52.972 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:31:53.120 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:31:54.272 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:31:57.810 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:31:57.935 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:02.642 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:24.047 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:24.047 INFO:teuthology.orchestra.run.smithi196.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-05T02:32:24.047 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:32:24.090 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:24.452 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:24.453 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:32:24.453 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:32:24.453 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:32:24.869 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:24.869 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:32:24.869 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:32:24.869 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-05T02:32:24.869 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-05T02:32:24.869 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-05T02:32:24.870 INFO:teuthology.orchestra.run.smithi196.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-05T02:32:24.870 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:32:24.870 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:32:25.184 DEBUG:teuthology.orchestra.run.smithi196:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:32:25.275 INFO:teuthology.orchestra.run.smithi196.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:32:25.275 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:32:25.275 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-05T02:32:25.275 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-05T02:32:25.275 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-05T02:32:25.275 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-05T02:32:25.275 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi196.front.sepia.ceph.com, path=None, version=distro) 2024-04-05T02:32:25.276 DEBUG:teuthology.orchestra.run.smithi196:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:32:25.357 INFO:teuthology.orchestra.run.smithi196.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:32:25.357 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:32:25.357 DEBUG:teuthology.orchestra.run.smithi196:> sudo rpm -qi grub2-tools 2024-04-05T02:32:25.391 INFO:teuthology.orchestra.run.smithi196.stdout:Name : grub2-tools 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Epoch : 1 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Version : 2.06 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Release : 68.el9 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Architecture: x86_64 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Group : Unspecified 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Size : 8264254 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:License : GPLv3+ 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-05T02:32:25.392 INFO:teuthology.orchestra.run.smithi196.stdout:Packager : builder@centos.org 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout:Vendor : CentOS 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout:Summary : Support tools for GRUB. 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout:Description : 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-05T02:32:25.393 INFO:teuthology.orchestra.run.smithi196.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-05T02:32:25.394 INFO:teuthology.orchestra.run.smithi196.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-05T02:32:25.394 INFO:teuthology.orchestra.run.smithi196.stdout:hardware devices. 2024-04-05T02:32:25.394 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:32:25.394 INFO:teuthology.orchestra.run.smithi196.stdout:This subpackage provides tools for support of all platforms. 2024-04-05T02:32:25.395 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-05T02:32:25.395 INFO:teuthology.task.kernel:Updating grub on smithi196 to boot 5.14.0-435.el9.x86_64 2024-04-05T02:32:25.395 DEBUG:teuthology.orchestra.run.smithi196:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-05T02:32:25.993 INFO:teuthology.orchestra.run.smithi196.stderr:Generating grub configuration file ... 2024-04-05T02:32:27.309 INFO:teuthology.orchestra.run.smithi196.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-05T02:32:27.344 INFO:teuthology.orchestra.run.smithi196.stderr:done 2024-04-05T02:32:27.347 DEBUG:teuthology.orchestra.run.smithi196:> mktemp 2024-04-05T02:32:27.362 INFO:teuthology.orchestra.run.smithi196.stdout:/tmp/tmp.EtvJuPgU9m 2024-04-05T02:32:27.363 DEBUG:teuthology.orchestra.run.smithi196:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.EtvJuPgU9m 2024-04-05T02:32:27.431 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod 0666 /tmp/tmp.EtvJuPgU9m 2024-04-05T02:32:27.568 DEBUG:teuthology.orchestra.remote:smithi196:/tmp/tmp.EtvJuPgU9m is 6KB 2024-04-05T02:32:27.617 DEBUG:teuthology.orchestra.run.smithi196:> rm -fr /tmp/tmp.EtvJuPgU9m 2024-04-05T02:32:27.632 DEBUG:teuthology.orchestra.run.smithi196:> sudo /bin/ls /boot/loader/entries || true 2024-04-05T02:32:27.699 INFO:teuthology.orchestra.run.smithi196.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-05T02:32:27.699 INFO:teuthology.orchestra.run.smithi196.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-05T02:32:27.699 INFO:teuthology.orchestra.run.smithi196.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-05T02:32:27.699 INFO:teuthology.orchestra.run.smithi196.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-05T02:32:27.700 DEBUG:teuthology.orchestra.run.smithi196:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-05T02:32:27.849 DEBUG:teuthology.orchestra.run.smithi196:> sudo shutdown -r now 2024-04-05T02:32:32.173 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:32.173 INFO:teuthology.orchestra.run.smithi150.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-05T02:32:32.173 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:32:32.217 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-modules-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:32.595 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: kernel-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:32.595 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-5.14.0-435.el9.x86_64 1/4 2024-04-05T02:32:32.595 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-core-5.14.0-435.el9.x86_64 2/4 2024-04-05T02:32:32.595 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-modules-5.14.0-435.el9.x86_64 3/4 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : kernel-modules-core-5.14.0-435.el9.x86_64 4/4 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-5.14.0-435.el9.x86_64 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-core-5.14.0-435.el9.x86_64 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules-5.14.0-435.el9.x86_64 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: kernel-modules-core-5.14.0-435.el9.x86_64 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:32:33.022 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:32:33.369 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:32:33.472 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:32:33.473 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:32:33.473 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-435.el9.x86_64 2024-04-05T02:32:33.473 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-05T02:32:33.473 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-05T02:32:33.473 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-05T02:32:33.473 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi150.front.sepia.ceph.com, path=None, version=distro) 2024-04-05T02:32:33.473 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q kernel | sort -rV | head -n 1 2024-04-05T02:32:33.554 INFO:teuthology.orchestra.run.smithi150.stdout:kernel-5.14.0-435.el9.x86_64 2024-04-05T02:32:33.554 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-435.el9.x86_64 2024-04-05T02:32:33.554 DEBUG:teuthology.orchestra.run.smithi150:> sudo rpm -qi grub2-tools 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Name : grub2-tools 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Epoch : 1 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Version : 2.06 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Release : 68.el9 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Architecture: x86_64 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Group : Unspecified 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Size : 8264254 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:License : GPLv3+ 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-05T02:32:33.588 INFO:teuthology.orchestra.run.smithi150.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:Packager : builder@centos.org 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:Vendor : CentOS 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:Summary : Support tools for GRUB. 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:Description : 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:hardware devices. 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:32:33.589 INFO:teuthology.orchestra.run.smithi150.stdout:This subpackage provides tools for support of all platforms. 2024-04-05T02:32:33.590 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-05T02:32:33.590 INFO:teuthology.task.kernel:Updating grub on smithi150 to boot 5.14.0-435.el9.x86_64 2024-04-05T02:32:33.590 DEBUG:teuthology.orchestra.run.smithi150:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-05T02:32:34.256 INFO:teuthology.orchestra.run.smithi150.stderr:Generating grub configuration file ... 2024-04-05T02:32:35.601 INFO:teuthology.orchestra.run.smithi150.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-05T02:32:35.634 INFO:teuthology.orchestra.run.smithi150.stderr:done 2024-04-05T02:32:35.636 DEBUG:teuthology.orchestra.run.smithi150:> mktemp 2024-04-05T02:32:35.651 INFO:teuthology.orchestra.run.smithi150.stdout:/tmp/tmp.Wt6lXDH6DS 2024-04-05T02:32:35.651 DEBUG:teuthology.orchestra.run.smithi150:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Wt6lXDH6DS 2024-04-05T02:32:35.719 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0666 /tmp/tmp.Wt6lXDH6DS 2024-04-05T02:32:35.853 DEBUG:teuthology.orchestra.remote:smithi150:/tmp/tmp.Wt6lXDH6DS is 6KB 2024-04-05T02:32:35.902 DEBUG:teuthology.orchestra.run.smithi150:> rm -fr /tmp/tmp.Wt6lXDH6DS 2024-04-05T02:32:35.916 DEBUG:teuthology.orchestra.run.smithi150:> sudo /bin/ls /boot/loader/entries || true 2024-04-05T02:32:35.981 INFO:teuthology.orchestra.run.smithi150.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-05T02:32:35.981 INFO:teuthology.orchestra.run.smithi150.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-05T02:32:35.981 INFO:teuthology.orchestra.run.smithi150.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-05T02:32:35.981 INFO:teuthology.orchestra.run.smithi150.stdout:15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64.conf 2024-04-05T02:32:35.982 DEBUG:teuthology.orchestra.run.smithi150:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-435.el9.x86_64 2024-04-05T02:32:36.137 DEBUG:teuthology.orchestra.run.smithi150:> sudo shutdown -r now 2024-04-05T02:32:57.873 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-05T02:32:57.873 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-04-05T02:32:57.874 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:33:06.142 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-05T02:33:06.142 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:33:06.143 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:33:16.322 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.196 2024-04-05T02:33:24.510 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.150 2024-04-05T02:33:25.323 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-04-05T02:33:25.324 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:33:28.386 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.196 2024-04-05T02:33:33.514 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:33:33.515 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:33:36.578 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.150 2024-04-05T02:33:40.390 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-04-05T02:33:40.391 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:33:48.586 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:33:48.587 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:34:12.275 DEBUG:teuthology.orchestra.run.smithi196:> true 2024-04-05T02:34:12.664 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-04-05T02:34:12.664 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-435.el9.x86_64"... 2024-04-05T02:34:12.664 DEBUG:teuthology.orchestra.run.smithi196:> uname -r 2024-04-05T02:34:12.678 INFO:teuthology.orchestra.run.smithi196.stdout:5.14.0-435.el9.x86_64 2024-04-05T02:34:12.678 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-05T02:34:12.678 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-05T02:34:12.678 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-05T02:34:13.680 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-05T02:34:13.680 DEBUG:teuthology.orchestra.run.smithi196:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-05T02:34:13.802 INFO:teuthology.orchestra.run.smithi196.stdout:ttyS1 2024-04-05T02:34:13.847 DEBUG:teuthology.parallel:result is None 2024-04-05T02:34:48.589 DEBUG:teuthology.orchestra.remote:timed out 2024-04-05T02:35:03.590 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:35:03.591 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:35:03.869 DEBUG:teuthology.orchestra.run.smithi150:> true 2024-04-05T02:35:04.226 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:35:04.226 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-435.el9.x86_64"... 2024-04-05T02:35:04.226 DEBUG:teuthology.orchestra.run.smithi150:> uname -r 2024-04-05T02:35:04.241 INFO:teuthology.orchestra.run.smithi150.stdout:5.14.0-435.el9.x86_64 2024-04-05T02:35:04.241 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-435.el9.x86_64 vs 5.14.0-435.el9.x86_64 2024-04-05T02:35:04.241 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-05T02:35:04.241 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-05T02:35:05.242 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-05T02:35:05.243 DEBUG:teuthology.orchestra.run.smithi150:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-05T02:35:05.325 INFO:teuthology.orchestra.run.smithi150.stdout:ttyS1 2024-04-05T02:35:05.356 DEBUG:teuthology.parallel:result is None 2024-04-05T02:35:05.357 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-05T02:35:05.364 INFO:teuthology.task.internal:Creating test directory... 2024-04-05T02:35:05.364 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-05T02:35:05.366 DEBUG:teuthology.orchestra.run.smithi196:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-05T02:35:05.385 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-05T02:35:05.405 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-05T02:35:05.438 INFO:teuthology.task.internal:Creating archive directory... 2024-04-05T02:35:05.441 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-05T02:35:05.444 DEBUG:teuthology.orchestra.run.smithi196:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-05T02:35:05.472 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-05T02:35:05.478 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-05T02:35:05.478 DEBUG:teuthology.orchestra.run.smithi150:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-05T02:35:05.509 DEBUG:teuthology.orchestra.run.smithi196:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-05T02:35:05.546 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:35:05.547 INFO:teuthology.orchestra.run.smithi196.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:35:05.560 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:35:05.562 INFO:teuthology.orchestra.run.smithi196.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-05T02:35:05.564 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-05T02:35:05.571 INFO:teuthology.task.internal:Configuring sudo... 2024-04-05T02:35:05.571 DEBUG:teuthology.orchestra.run.smithi150:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-05T02:35:05.606 DEBUG:teuthology.orchestra.run.smithi196:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-05T02:35:05.636 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-05T02:35:05.645 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-05T02:35:05.646 DEBUG:teuthology.orchestra.run.smithi150:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-05T02:35:05.676 DEBUG:teuthology.orchestra.run.smithi196:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-05T02:35:05.693 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:35:05.773 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:35:05.822 DEBUG:teuthology.orchestra.run.smithi150:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:35:05.904 DEBUG:teuthology.orchestra.run.smithi150:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:35:05.948 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:35:05.948 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-05T02:35:06.017 DEBUG:teuthology.orchestra.run.smithi196:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:35:06.063 DEBUG:teuthology.orchestra.run.smithi196:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-05T02:35:06.124 DEBUG:teuthology.orchestra.run.smithi196:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:35:06.167 DEBUG:teuthology.orchestra.run.smithi196:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-05T02:35:06.213 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:35:06.213 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-05T02:35:06.286 DEBUG:teuthology.orchestra.run.smithi150:> sudo service rsyslog restart 2024-04-05T02:35:06.288 DEBUG:teuthology.orchestra.run.smithi196:> sudo service rsyslog restart 2024-04-05T02:35:06.330 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T02:35:06.381 INFO:teuthology.orchestra.run.smithi196.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T02:35:06.763 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-05T02:35:06.770 INFO:teuthology.task.internal:Starting timer... 2024-04-05T02:35:06.771 INFO:teuthology.run_tasks:Running task pcp... 2024-04-05T02:35:06.808 INFO:teuthology.run_tasks:Running task selinux... 2024-04-05T02:35:06.840 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-04-05T02:35:06.840 DEBUG:teuthology.orchestra.run.smithi150:> sudo service auditd rotate 2024-04-05T02:35:06.891 INFO:teuthology.orchestra.run.smithi150.stdout:Rotating logs: 2024-04-05T02:35:06.893 DEBUG:teuthology.orchestra.run.smithi196:> sudo service auditd rotate 2024-04-05T02:35:06.974 INFO:teuthology.orchestra.run.smithi196.stdout:Rotating logs: 2024-04-05T02:35:06.976 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-05T02:35:06.976 DEBUG:teuthology.orchestra.run.smithi150:> /usr/sbin/getenforce 2024-04-05T02:35:06.998 INFO:teuthology.orchestra.run.smithi150.stdout:Permissive 2024-04-05T02:35:06.998 DEBUG:teuthology.orchestra.run.smithi196:> /usr/sbin/getenforce 2024-04-05T02:35:07.017 INFO:teuthology.orchestra.run.smithi196.stdout:Permissive 2024-04-05T02:35:07.017 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi150.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi196.front.sepia.ceph.com': 'permissive'} 2024-04-05T02:35:07.017 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T02:35:07.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:35:07.068 DEBUG:teuthology.orchestra.run.smithi196:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T02:35:07.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T02:35:07.096 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-05T02:35:07.096 DEBUG:teuthology.orchestra.run.smithi150:> sudo /usr/sbin/setenforce permissive 2024-04-05T02:35:07.142 DEBUG:teuthology.orchestra.run.smithi196:> sudo /usr/sbin/setenforce permissive 2024-04-05T02:35:07.177 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-05T02:35:07.188 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-05T02:35:07.204 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-05T02:35:07.206 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi150.front.sepia.ceph.com,smithi196.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-05T02:39:20.293 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi150.front.sepia.ceph.com'), Remote(name='ubuntu@smithi196.front.sepia.ceph.com')] 2024-04-05T02:39:20.294 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:39:20.295 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi150.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:39:20.386 DEBUG:teuthology.orchestra.run.smithi150:> true 2024-04-05T02:39:20.469 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi150.front.sepia.ceph.com' 2024-04-05T02:39:20.470 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-04-05T02:39:20.470 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi196.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-05T02:39:20.562 DEBUG:teuthology.orchestra.run.smithi196:> true 2024-04-05T02:39:20.642 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi196.front.sepia.ceph.com' 2024-04-05T02:39:20.643 INFO:teuthology.run_tasks:Running task clock... 2024-04-05T02:39:20.655 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-05T02:39:20.655 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-05T02:39:20.655 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T02:39:20.658 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-05T02:39:20.658 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T02:39:20.686 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-05T02:39:20.705 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-05T02:39:20.718 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-05T02:39:20.736 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-05T02:39:20.738 INFO:teuthology.orchestra.run.smithi150.stderr:sudo: ntpd: command not found 2024-04-05T02:39:20.752 INFO:teuthology.orchestra.run.smithi150.stdout:506 Cannot talk to daemon 2024-04-05T02:39:20.767 INFO:teuthology.orchestra.run.smithi196.stderr:sudo: ntpd: command not found 2024-04-05T02:39:20.768 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-05T02:39:20.781 INFO:teuthology.orchestra.run.smithi196.stdout:506 Cannot talk to daemon 2024-04-05T02:39:20.785 INFO:teuthology.orchestra.run.smithi150.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-05T02:39:20.798 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-05T02:39:20.815 INFO:teuthology.orchestra.run.smithi196.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-05T02:39:20.835 INFO:teuthology.orchestra.run.smithi150.stderr:bash: line 1: ntpq: command not found 2024-04-05T02:39:20.838 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T02:39:20.838 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2024-04-05T02:39:20.838 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.838 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.838 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.838 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.868 INFO:teuthology.orchestra.run.smithi196.stderr:bash: line 1: ntpq: command not found 2024-04-05T02:39:20.871 INFO:teuthology.orchestra.run.smithi196.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T02:39:20.871 INFO:teuthology.orchestra.run.smithi196.stdout:=============================================================================== 2024-04-05T02:39:20.871 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.871 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.871 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.871 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T02:39:20.872 INFO:teuthology.run_tasks:Running task pexec... 2024-04-05T02:39:20.883 INFO:teuthology.task.pexec:Executing custom commands... 2024-04-05T02:39:20.884 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi150.front.sepia.ceph.com 2024-04-05T02:39:20.884 DEBUG:teuthology.orchestra.run.smithi150:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-05T02:39:20.884 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi196.front.sepia.ceph.com 2024-04-05T02:39:20.885 DEBUG:teuthology.orchestra.run.smithi196:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-04-05T02:39:21.469 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:50 ago on Fri 05 Apr 2024 02:38:31 AM UTC. 2024-04-05T02:39:21.498 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:50 ago on Fri 05 Apr 2024 02:38:31 AM UTC. 2024-04-05T02:39:21.617 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:39:21.639 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:39:21.640 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:21.640 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 3.1 M 2024-04-05T02:39:21.640 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 9.8 M 2024-04-05T02:39:21.640 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:39:21.646 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:39:21.647 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:39:21.647 INFO:teuthology.orchestra.run.smithi196.stdout: Package Architecture Version Repository Size 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout: runc x86_64 4:1.1.12-2.el9 appstream 3.1 M 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:39:21.648 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:39:21.649 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 3.1 M 2024-04-05T02:39:21.649 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 9.8 M 2024-04-05T02:39:21.649 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:39:22.540 INFO:teuthology.orchestra.run.smithi150.stdout:runc-1.1.12-2.el9.x86_64.rpm 5.0 MB/s | 3.1 MB 00:00 2024-04-05T02:39:22.541 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:39:22.541 INFO:teuthology.orchestra.run.smithi150.stdout:Total 3.5 MB/s | 3.1 MB 00:00 2024-04-05T02:39:22.578 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:39:22.584 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:39:22.584 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:39:22.663 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:39:22.664 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:39:22.872 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:39:22.977 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-05T02:39:23.290 INFO:teuthology.orchestra.run.smithi196.stdout:runc-1.1.12-2.el9.x86_64.rpm 2.3 MB/s | 3.1 MB 00:01 2024-04-05T02:39:23.290 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:39:23.290 INFO:teuthology.orchestra.run.smithi196.stdout:Total 1.9 MB/s | 3.1 MB 00:01 2024-04-05T02:39:23.328 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:39:23.334 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:39:23.334 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:39:23.417 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:39:23.417 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:39:23.489 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-05T02:39:23.635 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:39:23.718 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-05T02:39:23.777 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-05T02:39:23.777 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:23.777 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:39:23.777 INFO:teuthology.orchestra.run.smithi150.stdout: runc-4:1.1.12-2.el9.x86_64 2024-04-05T02:39:23.778 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:23.778 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:39:24.021 DEBUG:teuthology.parallel:result is None 2024-04-05T02:39:24.280 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-05T02:39:24.598 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : runc-4:1.1.12-2.el9.x86_64 1/1 2024-04-05T02:39:24.599 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:39:24.599 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:39:24.599 INFO:teuthology.orchestra.run.smithi196.stdout: runc-4:1.1.12-2.el9.x86_64 2024-04-05T02:39:24.599 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:39:24.599 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:39:24.862 DEBUG:teuthology.parallel:result is None 2024-04-05T02:39:24.862 INFO:teuthology.run_tasks:Running task install... 2024-04-05T02:39:24.871 DEBUG:teuthology.task.install:project ceph 2024-04-05T02:39:24.871 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06'}} 2024-04-05T02:39:24.871 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06'} 2024-04-05T02:39:24.871 INFO:teuthology.task.install:Using flavor: default 2024-04-05T02:39:24.877 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-05T02:39:24.877 INFO:teuthology.task.install:extra packages: [] 2024-04-05T02:39:24.878 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06', 'tag': None, 'wait_for_package': False} 2024-04-05T02:39:24.878 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:39:24.878 INFO:teuthology.packaging:ref: None 2024-04-05T02:39:24.878 INFO:teuthology.packaging:tag: None 2024-04-05T02:39:24.878 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:39:24.878 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:39:24.878 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:39:24.880 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06', 'tag': None, 'wait_for_package': False} 2024-04-05T02:39:24.880 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:39:24.880 INFO:teuthology.packaging:ref: None 2024-04-05T02:39:24.880 INFO:teuthology.packaging:tag: None 2024-04-05T02:39:24.880 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:39:24.880 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:39:24.881 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:39:25.111 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/ 2024-04-05T02:39:25.112 INFO:teuthology.task.install.rpm:Package version is 17.2.7-815.g93c790e1 2024-04-05T02:39:25.228 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-05T02:39:25.228 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:39:25.228 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-05T02:39:25.261 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-05T02:39:25.261 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:39:25.261 INFO:teuthology.packaging:ref: None 2024-04-05T02:39:25.261 INFO:teuthology.packaging:tag: None 2024-04-05T02:39:25.261 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:39:25.261 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:39:25.262 DEBUG:teuthology.orchestra.run.smithi150:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-05T02:39:25.327 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/ 2024-04-05T02:39:25.328 INFO:teuthology.task.install.rpm:Package version is 17.2.7-815.g93c790e1 2024-04-05T02:39:25.333 DEBUG:teuthology.orchestra.run.smithi150:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-05T02:39:25.420 DEBUG:teuthology.orchestra.run.smithi150:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-05T02:39:25.448 INFO:teuthology.orchestra.run.smithi150.stdout:check_obsoletes = 1 2024-04-05T02:39:25.450 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum clean all 2024-04-05T02:39:25.508 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/93c790e135cc94d629f832b908e3f6c70360bfd5/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-05T02:39:25.508 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:39:25.508 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-05T02:39:25.541 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-05T02:39:25.541 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:39:25.541 INFO:teuthology.packaging:ref: None 2024-04-05T02:39:25.541 INFO:teuthology.packaging:tag: None 2024-04-05T02:39:25.542 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:39:25.542 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:39:25.542 DEBUG:teuthology.orchestra.run.smithi196:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-05T02:39:25.609 DEBUG:teuthology.orchestra.run.smithi196:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-05T02:39:25.689 DEBUG:teuthology.orchestra.run.smithi196:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-05T02:39:25.723 INFO:teuthology.orchestra.run.smithi150.stdout:74 files removed 2024-04-05T02:39:25.751 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-radosgw 2024-04-05T02:39:25.756 INFO:teuthology.orchestra.run.smithi196.stdout:check_obsoletes = 1 2024-04-05T02:39:25.757 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum clean all 2024-04-05T02:39:26.003 INFO:teuthology.orchestra.run.smithi196.stdout:74 files removed 2024-04-05T02:39:26.030 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-radosgw 2024-04-05T02:39:26.557 INFO:teuthology.orchestra.run.smithi150.stdout:ceph packages for x86_64 153 kB/s | 77 kB 00:00 2024-04-05T02:39:26.796 INFO:teuthology.orchestra.run.smithi196.stdout:ceph packages for x86_64 166 kB/s | 77 kB 00:00 2024-04-05T02:39:27.049 INFO:teuthology.orchestra.run.smithi150.stdout:ceph noarch packages 21 kB/s | 9.2 kB 00:00 2024-04-05T02:39:27.238 INFO:teuthology.orchestra.run.smithi196.stdout:ceph noarch packages 23 kB/s | 9.2 kB 00:00 2024-04-05T02:39:27.491 INFO:teuthology.orchestra.run.smithi150.stdout:ceph source packages 4.4 kB/s | 1.8 kB 00:00 2024-04-05T02:39:27.555 INFO:teuthology.orchestra.run.smithi196.stdout:ceph source packages 6.4 kB/s | 1.8 kB 00:00 2024-04-05T02:39:28.310 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS-9 - AppStream 8.6 MB/s | 6.7 MB 00:00 2024-04-05T02:39:28.366 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS-9 - AppStream 8.7 MB/s | 6.7 MB 00:00 2024-04-05T02:39:29.752 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS-9 - BaseOS 6.2 MB/s | 2.0 MB 00:00 2024-04-05T02:39:29.941 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS-9 - BaseOS 4.4 MB/s | 2.0 MB 00:00 2024-04-05T02:39:30.876 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.0 MB 00:00 2024-04-05T02:39:31.873 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - BaseOS 5.0 MB/s | 8.0 MB 00:01 2024-04-05T02:39:32.978 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-05T02:39:33.691 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-05T02:39:37.104 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - CRB 9.8 MB/s | 6.1 MB 00:00 2024-04-05T02:39:38.738 INFO:teuthology.orchestra.run.smithi150.stdout:CentOS Stream 9 - Extras packages 60 kB/s | 16 kB 00:00 2024-04-05T02:39:38.800 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.1 MB 00:01 2024-04-05T02:39:40.092 INFO:teuthology.orchestra.run.smithi150.stdout:Extra Packages for Enterprise Linux 17 MB/s | 21 MB 00:01 2024-04-05T02:39:41.034 INFO:teuthology.orchestra.run.smithi196.stdout:CentOS Stream 9 - Extras packages 14 kB/s | 16 kB 00:01 2024-04-05T02:39:42.319 INFO:teuthology.orchestra.run.smithi196.stdout:Extra Packages for Enterprise Linux 18 MB/s | 21 MB 00:01 2024-04-05T02:39:45.073 INFO:teuthology.orchestra.run.smithi150.stdout:lab-extras 42 kB/s | 1.7 kB 00:00 2024-04-05T02:39:47.302 INFO:teuthology.orchestra.run.smithi196.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-04-05T02:39:47.673 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-radosgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 14 M 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout:Upgrading: 2024-04-05T02:39:47.675 INFO:teuthology.orchestra.run.smithi150.stdout: librados2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.1 M 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: librbd1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-base x86_64 2:17.2.7-815.g93c790e1.el9 ceph 5.0 M 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 22 M 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-selinux x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 637 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: libradosstriper1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 458 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: librgw2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 4.9 M 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-05T02:39:47.676 INFO:teuthology.orchestra.run.smithi150.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-argparse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 45 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 117 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cephfs x86_64 2:17.2.7-815.g93c790e1.el9 ceph 156 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rados x86_64 2:17.2.7-815.g93c790e1.el9 ceph 320 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 293 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 99 k 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout:Install 22 Packages 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout:Upgrade 2 Packages 2024-04-05T02:39:47.677 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:47.678 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 58 M 2024-04-05T02:39:47.678 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:39:48.457 INFO:teuthology.orchestra.run.smithi150.stdout:(1/24): ceph-base-17.2.7-815.g93c790e1.el9.x86_ 7.2 MB/s | 5.0 MB 00:00 2024-04-05T02:39:48.516 INFO:teuthology.orchestra.run.smithi150.stdout:(2/24): ceph-selinux-17.2.7-815.g93c790e1.el9.x 431 kB/s | 25 kB 00:00 2024-04-05T02:39:48.625 INFO:teuthology.orchestra.run.smithi150.stdout:(3/24): libcephfs2-17.2.7-815.g93c790e1.el9.x86 5.8 MB/s | 637 kB 00:00 2024-04-05T02:39:48.708 INFO:teuthology.orchestra.run.smithi150.stdout:(4/24): libradosstriper1-17.2.7-815.g93c790e1.e 5.4 MB/s | 458 kB 00:00 2024-04-05T02:39:49.234 INFO:teuthology.orchestra.run.smithi150.stdout:(5/24): ceph-radosgw-17.2.7-815.g93c790e1.el9.x 9.6 MB/s | 14 MB 00:01 2024-04-05T02:39:49.350 INFO:teuthology.orchestra.run.smithi150.stdout:(6/24): librgw2-17.2.7-815.g93c790e1.el9.x86_64 7.6 MB/s | 4.9 MB 00:00 2024-04-05T02:39:49.376 INFO:teuthology.orchestra.run.smithi150.stdout:(7/24): python3-ceph-argparse-17.2.7-815.g93c79 318 kB/s | 45 kB 00:00 2024-04-05T02:39:49.401 INFO:teuthology.orchestra.run.smithi150.stdout:(8/24): python3-ceph-common-17.2.7-815.g93c790e 2.3 MB/s | 117 kB 00:00 2024-04-05T02:39:49.426 INFO:teuthology.orchestra.run.smithi150.stdout:(9/24): python3-cephfs-17.2.7-815.g93c790e1.el9 3.0 MB/s | 156 kB 00:00 2024-04-05T02:39:49.460 INFO:teuthology.orchestra.run.smithi150.stdout:(10/24): python3-rados-17.2.7-815.g93c790e1.el9 5.4 MB/s | 320 kB 00:00 2024-04-05T02:39:49.486 INFO:teuthology.orchestra.run.smithi150.stdout:(11/24): python3-rbd-17.2.7-815.g93c790e1.el9.x 4.9 MB/s | 293 kB 00:00 2024-04-05T02:39:49.510 INFO:teuthology.orchestra.run.smithi150.stdout:(12/24): python3-rgw-17.2.7-815.g93c790e1.el9.x 1.9 MB/s | 99 kB 00:00 2024-04-05T02:39:49.628 INFO:teuthology.orchestra.run.smithi150.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 478 kB/s | 44 kB 00:00 2024-04-05T02:39:49.652 INFO:teuthology.orchestra.run.smithi150.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 952 kB/s | 159 kB 00:00 2024-04-05T02:39:49.694 INFO:teuthology.orchestra.run.smithi150.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 9.7 MB/s | 661 kB 00:00 2024-04-05T02:39:49.720 INFO:teuthology.orchestra.run.smithi150.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.1 MB/s | 282 kB 00:00 2024-04-05T02:39:49.737 INFO:teuthology.orchestra.run.smithi150.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 987 kB/s | 41 kB 00:00 2024-04-05T02:39:49.887 INFO:teuthology.orchestra.run.smithi150.stdout:(18/24): liboath-2.6.7-2.el9.x86_64.rpm 324 kB/s | 49 kB 00:00 2024-04-05T02:39:49.939 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-radosgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 14 M 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout:Upgrading: 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout: librados2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.1 M 2024-04-05T02:39:49.941 INFO:teuthology.orchestra.run.smithi196.stdout: librbd1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-base x86_64 2:17.2.7-815.g93c790e1.el9 ceph 5.0 M 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 22 M 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-selinux x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: libcephfs2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 637 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: libradosstriper1 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 458 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: librgw2 x86_64 2:17.2.7-815.g93c790e1.el9 ceph 4.9 M 2024-04-05T02:39:49.942 INFO:teuthology.orchestra.run.smithi196.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-ceph-argparse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 45 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-ceph-common x86_64 2:17.2.7-815.g93c790e1.el9 ceph 117 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cephfs x86_64 2:17.2.7-815.g93c790e1.el9 ceph 156 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rados x86_64 2:17.2.7-815.g93c790e1.el9 ceph 320 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 293 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rgw x86_64 2:17.2.7-815.g93c790e1.el9 ceph 99 k 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout:Install 22 Packages 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout:Upgrade 2 Packages 2024-04-05T02:39:49.943 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:39:49.944 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 58 M 2024-04-05T02:39:49.944 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:39:49.996 INFO:teuthology.orchestra.run.smithi150.stdout:(19/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.1 MB/s | 309 kB 00:00 2024-04-05T02:39:50.021 INFO:teuthology.orchestra.run.smithi150.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 505 kB/s | 67 kB 00:00 2024-04-05T02:39:50.238 INFO:teuthology.orchestra.run.smithi150.stdout:(21/24): ceph-common-17.2.7-815.g93c790e1.el9.x 9.1 MB/s | 22 MB 00:02 2024-04-05T02:39:50.346 INFO:teuthology.orchestra.run.smithi150.stdout:(22/24): librados2-17.2.7-815.g93c790e1.el9.x86 9.5 MB/s | 3.1 MB 00:00 2024-04-05T02:39:50.380 INFO:teuthology.orchestra.run.smithi150.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 4.4 MB/s | 1.6 MB 00:00 2024-04-05T02:39:50.564 INFO:teuthology.orchestra.run.smithi150.stdout:(24/24): librbd1-17.2.7-815.g93c790e1.el9.x86_6 9.0 MB/s | 2.9 MB 00:00 2024-04-05T02:39:50.567 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:39:50.567 INFO:teuthology.orchestra.run.smithi150.stdout:Total 20 MB/s | 58 MB 00:02 2024-04-05T02:39:50.568 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:39:50.597 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:39:50.597 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:39:51.055 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:39:51.056 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:39:51.744 INFO:teuthology.orchestra.run.smithi196.stdout:(1/24): ceph-common-17.2.7-815.g93c790e1.el9.x8 13 MB/s | 22 MB 00:01 2024-04-05T02:39:51.811 INFO:teuthology.orchestra.run.smithi196.stdout:(2/24): ceph-base-17.2.7-815.g93c790e1.el9.x86_ 2.8 MB/s | 5.0 MB 00:01 2024-04-05T02:39:51.836 INFO:teuthology.orchestra.run.smithi196.stdout:(3/24): ceph-selinux-17.2.7-815.g93c790e1.el9.x 274 kB/s | 25 kB 00:00 2024-04-05T02:39:51.895 INFO:teuthology.orchestra.run.smithi196.stdout:(4/24): libradosstriper1-17.2.7-815.g93c790e1.e 7.7 MB/s | 458 kB 00:00 2024-04-05T02:39:51.987 INFO:teuthology.orchestra.run.smithi196.stdout:(5/24): libcephfs2-17.2.7-815.g93c790e1.el9.x86 3.5 MB/s | 637 kB 00:00 2024-04-05T02:39:52.071 INFO:teuthology.orchestra.run.smithi196.stdout:(6/24): python3-ceph-argparse-17.2.7-815.g93c79 543 kB/s | 45 kB 00:00 2024-04-05T02:39:52.170 INFO:teuthology.orchestra.run.smithi196.stdout:(7/24): python3-ceph-common-17.2.7-815.g93c790e 1.2 MB/s | 117 kB 00:00 2024-04-05T02:39:52.229 INFO:teuthology.orchestra.run.smithi196.stdout:(8/24): librgw2-17.2.7-815.g93c790e1.el9.x86_64 15 MB/s | 4.9 MB 00:00 2024-04-05T02:39:52.244 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:39:52.254 INFO:teuthology.orchestra.run.smithi196.stdout:(9/24): python3-cephfs-17.2.7-815.g93c790e1.el9 1.8 MB/s | 156 kB 00:00 2024-04-05T02:39:52.280 INFO:teuthology.orchestra.run.smithi196.stdout:(10/24): python3-rados-17.2.7-815.g93c790e1.el9 6.3 MB/s | 320 kB 00:00 2024-04-05T02:39:52.322 INFO:teuthology.orchestra.run.smithi196.stdout:(11/24): python3-rgw-17.2.7-815.g93c790e1.el9.x 2.3 MB/s | 99 kB 00:00 2024-04-05T02:39:52.347 INFO:teuthology.orchestra.run.smithi196.stdout:(12/24): python3-rbd-17.2.7-815.g93c790e1.el9.x 3.1 MB/s | 293 kB 00:00 2024-04-05T02:39:52.464 INFO:teuthology.orchestra.run.smithi196.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 378 kB/s | 44 kB 00:00 2024-04-05T02:39:52.481 INFO:teuthology.orchestra.run.smithi196.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.0 MB/s | 159 kB 00:00 2024-04-05T02:39:52.523 INFO:teuthology.orchestra.run.smithi196.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 661 kB 00:00 2024-04-05T02:39:52.548 INFO:teuthology.orchestra.run.smithi196.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.1 MB/s | 282 kB 00:00 2024-04-05T02:39:52.573 INFO:teuthology.orchestra.run.smithi196.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 826 kB/s | 41 kB 00:00 2024-04-05T02:39:52.576 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-05T02:39:52.624 INFO:teuthology.orchestra.run.smithi150.stdout: Upgrading : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:39:52.700 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:39:52.716 INFO:teuthology.orchestra.run.smithi196.stdout:(18/24): ceph-radosgw-17.2.7-815.g93c790e1.el9. 5.2 MB/s | 14 MB 00:02 2024-04-05T02:39:52.777 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:39:52.826 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:39:52.914 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:39:52.957 INFO:teuthology.orchestra.run.smithi196.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 127 kB/s | 49 kB 00:00 2024-04-05T02:39:52.981 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-05T02:39:53.022 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-05T02:39:53.057 INFO:teuthology.orchestra.run.smithi196.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 197 kB/s | 67 kB 00:00 2024-04-05T02:39:53.166 INFO:teuthology.orchestra.run.smithi196.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 500 kB/s | 309 kB 00:00 2024-04-05T02:39:53.366 INFO:teuthology.orchestra.run.smithi196.stdout:(22/24): librbd1-17.2.7-815.g93c790e1.el9.x86_6 15 MB/s | 2.9 MB 00:00 2024-04-05T02:39:53.455 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-05T02:39:53.474 INFO:teuthology.orchestra.run.smithi196.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 3.1 MB/s | 1.6 MB 00:00 2024-04-05T02:39:53.487 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:39:53.550 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:39:53.578 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-05T02:39:53.621 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 10/26 2024-04-05T02:39:53.641 INFO:teuthology.orchestra.run.smithi196.stdout:(24/24): librados2-17.2.7-815.g93c790e1.el9.x86 5.3 MB/s | 3.1 MB 00:00 2024-04-05T02:39:53.646 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:39:53.646 INFO:teuthology.orchestra.run.smithi196.stdout:Total 16 MB/s | 58 MB 00:03 2024-04-05T02:39:53.646 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:39:53.668 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:39:53.678 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:39:53.678 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:39:53.737 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:39:53.765 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:39:53.814 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:39:53.853 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-05T02:39:53.889 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-05T02:39:53.922 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-05T02:39:54.142 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:39:54.142 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:39:54.223 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-05T02:39:54.260 INFO:teuthology.orchestra.run.smithi150.stdout: Upgrading : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:39:54.355 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:39:54.431 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 19/26 2024-04-05T02:39:54.454 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 20/26 2024-04-05T02:39:55.317 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:39:55.657 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-05T02:39:55.719 INFO:teuthology.orchestra.run.smithi196.stdout: Upgrading : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:39:55.819 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:39:55.905 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:39:55.953 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:39:56.025 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:39:56.080 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-05T02:39:56.108 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-05T02:39:56.540 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-05T02:39:56.574 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:39:56.627 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 8/26 2024-04-05T02:39:56.656 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-05T02:39:56.699 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 10/26 2024-04-05T02:39:56.743 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:39:56.806 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:39:56.834 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:39:56.850 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:39:56.884 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:39:56.900 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:39:56.939 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-05T02:39:56.977 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-05T02:39:57.008 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-05T02:39:57.301 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-05T02:39:57.329 INFO:teuthology.orchestra.run.smithi196.stdout: Upgrading : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:39:57.406 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 18/26 2024-04-05T02:39:57.412 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:39:57.471 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:39:57.476 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 19/26 2024-04-05T02:39:57.507 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 20/26 2024-04-05T02:39:57.571 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:39:57.571 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-05T02:39:57.571 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-05T02:39:57.571 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:39:57.614 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:39:59.932 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:39:59.970 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:40:00.489 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 21/26 2024-04-05T02:40:00.523 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:40:00.604 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 22/26 2024-04-05T02:40:00.604 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-05T02:40:00.604 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-05T02:40:00.604 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:00.649 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:40:07.249 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:40:07.249 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /sys 2024-04-05T02:40:07.249 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /proc 2024-04-05T02:40:07.249 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /mnt 2024-04-05T02:40:07.250 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /var/tmp 2024-04-05T02:40:07.250 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /home 2024-04-05T02:40:07.250 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /root 2024-04-05T02:40:07.250 INFO:teuthology.orchestra.run.smithi150.stdout:skipping the directory /tmp 2024-04-05T02:40:07.250 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:08.556 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:40:08.617 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:40:08.617 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:08.617 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-05T02:40:08.617 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:40:08.617 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:40:08.617 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:08.619 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:40:08.710 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:40:08.710 INFO:teuthology.orchestra.run.smithi150.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:40:09.914 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:40:09.914 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 1/26 2024-04-05T02:40:09.914 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 5/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 6/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 7/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 8/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 9/26 2024-04-05T02:40:09.915 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 10/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-05T02:40:09.916 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:40:09.917 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-05T02:40:09.917 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 25/26 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /sys 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /proc 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /mnt 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /var/tmp 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /home 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /root 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout:skipping the directory /tmp 2024-04-05T02:40:10.278 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout:Upgraded: 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.846 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.847 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:10.848 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:11.205 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-test 2024-04-05T02:40:11.519 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:40:11.598 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 24/26 2024-04-05T02:40:11.598 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:11.598 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-05T02:40:11.598 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:40:11.598 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-05T02:40:11.598 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:11.599 INFO:teuthology.orchestra.run.smithi196.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:40:11.700 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-05T02:40:11.700 INFO:teuthology.orchestra.run.smithi196.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:40:11.772 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:26 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:11.925 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-test x86_64 2:17.2.7-815.g93c790e1.el9 ceph 48 M 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: libcephsqlite x86_64 2:17.2.7-815.g93c790e1.el9 ceph 163 k 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-04-05T02:40:11.926 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout:Install 6 Packages 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 49 M 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 193 M 2024-04-05T02:40:11.927 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:12.364 INFO:teuthology.orchestra.run.smithi150.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.2 MB/s | 217 kB 00:00 2024-04-05T02:40:12.414 INFO:teuthology.orchestra.run.smithi150.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.9 MB/s | 300 kB 00:00 2024-04-05T02:40:12.440 INFO:teuthology.orchestra.run.smithi150.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.5 MB/s | 63 kB 00:00 2024-04-05T02:40:12.498 INFO:teuthology.orchestra.run.smithi150.stdout:(4/6): libcephsqlite-17.2.7-815.g93c790e1.el9.x 535 kB/s | 163 kB 00:00 2024-04-05T02:40:12.927 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 1/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 3/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 4/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 5/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 6/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 7/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x 8/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86 9/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 10/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 11/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 12/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 13/26 2024-04-05T02:40:12.928 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librados2-2:17.2.7-815.g93c790e1.el9.x86_64 23/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-05T02:40:12.929 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 25/26 2024-04-05T02:40:13.340 INFO:teuthology.orchestra.run.smithi150.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 208 kB/s | 187 kB 00:00 2024-04-05T02:40:13.944 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-05T02:40:13.944 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:13.944 INFO:teuthology.orchestra.run.smithi196.stdout:Upgraded: 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: librados2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-radosgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-selinux-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: libradosstriper1-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.945 INFO:teuthology.orchestra.run.smithi196.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: librgw2-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-ceph-argparse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-ceph-common-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:13.946 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:14.266 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-test 2024-04-05T02:40:14.532 INFO:teuthology.orchestra.run.smithi150.stdout:(6/6): ceph-test-17.2.7-815.g93c790e1.el9.x86_6 20 MB/s | 48 MB 00:02 2024-04-05T02:40:14.535 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:14.535 INFO:teuthology.orchestra.run.smithi150.stdout:Total 19 MB/s | 49 MB 00:02 2024-04-05T02:40:14.550 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:40:14.572 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:40:14.572 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:40:14.827 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:27 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:14.954 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:40:14.955 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:40:14.978 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:14.978 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-test x86_64 2:17.2.7-815.g93c790e1.el9 ceph 48 M 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: libcephsqlite x86_64 2:17.2.7-815.g93c790e1.el9 ceph 163 k 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:40:14.979 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:14.980 INFO:teuthology.orchestra.run.smithi196.stdout:Install 6 Packages 2024-04-05T02:40:14.980 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:14.980 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 49 M 2024-04-05T02:40:14.980 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 193 M 2024-04-05T02:40:14.980 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:40:15.262 INFO:teuthology.orchestra.run.smithi196.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.4 MB/s | 217 kB 00:00 2024-04-05T02:40:15.312 INFO:teuthology.orchestra.run.smithi196.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.9 MB/s | 300 kB 00:00 2024-04-05T02:40:15.345 INFO:teuthology.orchestra.run.smithi196.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.8 MB/s | 63 kB 00:00 2024-04-05T02:40:15.446 INFO:teuthology.orchestra.run.smithi196.stdout:(4/6): jq-1.6-16.el9.x86_64.rpm 1.8 MB/s | 187 kB 00:00 2024-04-05T02:40:15.471 INFO:teuthology.orchestra.run.smithi196.stdout:(5/6): libcephsqlite-17.2.7-815.g93c790e1.el9.x 454 kB/s | 163 kB 00:00 2024-04-05T02:40:15.529 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:40:15.600 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-05T02:40:15.638 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-05T02:40:15.673 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:40:15.724 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-05T02:40:15.752 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:40:17.596 INFO:teuthology.orchestra.run.smithi196.stdout:(6/6): ceph-test-17.2.7-815.g93c790e1.el9.x86_6 19 MB/s | 48 MB 00:02 2024-04-05T02:40:17.599 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:17.599 INFO:teuthology.orchestra.run.smithi196.stdout:Total 19 MB/s | 49 MB 00:02 2024-04-05T02:40:17.614 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:40:17.634 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:40:17.635 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:40:18.014 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:40:18.015 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:40:18.584 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:40:18.654 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-05T02:40:18.702 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-05T02:40:18.738 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:40:18.779 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-05T02:40:18.807 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:40:20.166 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:40:20.300 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:40:22.523 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:40:22.524 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 1/6 2024-04-05T02:40:22.524 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2/6 2024-04-05T02:40:22.524 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:40:22.524 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-04-05T02:40:22.524 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-04-05T02:40:22.886 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: jq-1.6-16.el9.x86_64 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:22.887 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:23.145 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph 2024-04-05T02:40:23.201 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 5/6 2024-04-05T02:40:23.272 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:40:23.712 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:38 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:23.864 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================== 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================== 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout: ceph x86_64 2:17.2.7-815.g93c790e1.el9 ceph 6.5 k 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mds x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.1 M 2024-04-05T02:40:23.867 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr x86_64 2:17.2.7-815.g93c790e1.el9 ceph 1.4 M 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-modules-core noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 240 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mon x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.9 M 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-osd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 15 M 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-05T02:40:23.868 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-05T02:40:23.869 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================== 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout:Install 33 Packages 2024-04-05T02:40:23.870 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:23.871 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 27 M 2024-04-05T02:40:23.871 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 96 M 2024-04-05T02:40:23.871 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:24.444 INFO:teuthology.orchestra.run.smithi150.stdout:(1/33): ceph-17.2.7-815.g93c790e1.el9.x86_64.rp 41 kB/s | 6.5 kB 00:00 2024-04-05T02:40:24.811 INFO:teuthology.orchestra.run.smithi150.stdout:(2/33): ceph-mon-17.2.7-815.g93c790e1.el9.x86_6 11 MB/s | 3.9 MB 00:00 2024-04-05T02:40:25.020 INFO:teuthology.orchestra.run.smithi150.stdout:(3/33): ceph-mds-17.2.7-815.g93c790e1.el9.x86_6 2.8 MB/s | 2.1 MB 00:00 2024-04-05T02:40:25.137 INFO:teuthology.orchestra.run.smithi150.stdout:(4/33): ceph-mgr-modules-core-17.2.7-815.g93c79 2.0 MB/s | 240 kB 00:00 2024-04-05T02:40:25.221 INFO:teuthology.orchestra.run.smithi150.stdout:(5/33): ceph-mgr-17.2.7-815.g93c790e1.el9.x86_6 1.5 MB/s | 1.4 MB 00:00 2024-04-05T02:40:25.279 INFO:teuthology.orchestra.run.smithi150.stdout:(6/33): python3-mako-1.1.4-6.el9.noarch.rpm 1.0 MB/s | 144 kB 00:00 2024-04-05T02:40:25.304 INFO:teuthology.orchestra.run.smithi150.stdout:(7/33): python3-markupsafe-1.1.1-12.el9.x86_64. 384 kB/s | 32 kB 00:00 2024-04-05T02:40:25.329 INFO:teuthology.orchestra.run.smithi150.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 949 kB/s | 47 kB 00:00 2024-04-05T02:40:25.355 INFO:teuthology.orchestra.run.smithi150.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 774 kB/s | 39 kB 00:00 2024-04-05T02:40:25.380 INFO:teuthology.orchestra.run.smithi150.stdout:(10/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.7 MB/s | 241 kB 00:00 2024-04-05T02:40:25.439 INFO:teuthology.orchestra.run.smithi150.stdout:(11/33): python3-cryptography-36.0.1-4.el9.x86_ 14 MB/s | 1.1 MB 00:00 2024-04-05T02:40:25.455 INFO:teuthology.orchestra.run.smithi150.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 1.3 MB/s | 103 kB 00:00 2024-04-05T02:40:25.481 INFO:teuthology.orchestra.run.smithi150.stdout:(13/33): python3-pycparser-2.20-6.el9.noarch.rp 2.9 MB/s | 124 kB 00:00 2024-04-05T02:40:25.506 INFO:teuthology.orchestra.run.smithi150.stdout:(14/33): python3-urllib3-1.26.5-5.el9.noarch.rp 3.7 MB/s | 187 kB 00:00 2024-04-05T02:40:25.512 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 6/6 2024-04-05T02:40:25.512 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 1/6 2024-04-05T02:40:25.512 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2/6 2024-04-05T02:40:25.512 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-05T02:40:25.512 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-04-05T02:40:25.512 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-04-05T02:40:25.656 INFO:teuthology.orchestra.run.smithi150.stdout:(15/33): ceph-osd-17.2.7-815.g93c790e1.el9.x86_ 18 MB/s | 15 MB 00:00 2024-04-05T02:40:25.815 INFO:teuthology.orchestra.run.smithi150.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 374 kB/s | 125 kB 00:00 2024-04-05T02:40:25.850 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-test-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: jq-1.6-16.el9.x86_64 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: libcephsqlite-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-05T02:40:25.851 INFO:teuthology.orchestra.run.smithi196.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-05T02:40:25.853 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:25.853 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:25.857 INFO:teuthology.orchestra.run.smithi150.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 124 kB/s | 43 kB 00:00 2024-04-05T02:40:25.957 INFO:teuthology.orchestra.run.smithi150.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 572 kB/s | 172 kB 00:00 2024-04-05T02:40:25.974 INFO:teuthology.orchestra.run.smithi150.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 91 kB/s | 11 kB 00:00 2024-04-05T02:40:26.000 INFO:teuthology.orchestra.run.smithi150.stdout:(20/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.9 MB/s | 358 kB 00:00 2024-04-05T02:40:26.016 INFO:teuthology.orchestra.run.smithi150.stdout:(21/33): python3-jaraco-classes-3.2.1-5.el9.noa 300 kB/s | 18 kB 00:00 2024-04-05T02:40:26.058 INFO:teuthology.orchestra.run.smithi150.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 331 kB/s | 19 kB 00:00 2024-04-05T02:40:26.084 INFO:teuthology.orchestra.run.smithi150.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 293 kB/s | 20 kB 00:00 2024-04-05T02:40:26.101 INFO:teuthology.orchestra.run.smithi150.stdout:(24/33): python3-jaraco-collections-3.0.0-8.el9 184 kB/s | 23 kB 00:00 2024-04-05T02:40:26.117 INFO:teuthology.orchestra.run.smithi150.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 789 kB/s | 46 kB 00:00 2024-04-05T02:40:26.131 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph 2024-04-05T02:40:26.142 INFO:teuthology.orchestra.run.smithi150.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.3 MB/s | 79 kB 00:00 2024-04-05T02:40:26.176 INFO:teuthology.orchestra.run.smithi150.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 280 kB/s | 16 kB 00:00 2024-04-05T02:40:26.210 INFO:teuthology.orchestra.run.smithi150.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.3 MB/s | 90 kB 00:00 2024-04-05T02:40:26.244 INFO:teuthology.orchestra.run.smithi150.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 534 kB/s | 36 kB 00:00 2024-04-05T02:40:26.277 INFO:teuthology.orchestra.run.smithi150.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.5 MB/s | 272 kB 00:00 2024-04-05T02:40:26.353 INFO:teuthology.orchestra.run.smithi150.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.6 MB/s | 230 kB 00:00 2024-04-05T02:40:26.378 INFO:teuthology.orchestra.run.smithi150.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.1 MB/s | 427 kB 00:00 2024-04-05T02:40:26.403 INFO:teuthology.orchestra.run.smithi150.stdout:(33/33): python3-zc-lockfile-2.0-10.el9.noarch. 159 kB/s | 20 kB 00:00 2024-04-05T02:40:26.407 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:26.407 INFO:teuthology.orchestra.run.smithi150.stdout:Total 11 MB/s | 27 MB 00:02 2024-04-05T02:40:26.421 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:40:26.460 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:40:26.460 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:40:26.699 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:39 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:26.810 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:40:26.811 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:40:26.850 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:26.852 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================== 2024-04-05T02:40:26.852 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:40:26.852 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================== 2024-04-05T02:40:26.852 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:40:26.852 INFO:teuthology.orchestra.run.smithi196.stdout: ceph x86_64 2:17.2.7-815.g93c790e1.el9 ceph 6.5 k 2024-04-05T02:40:26.852 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mds x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.1 M 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr x86_64 2:17.2.7-815.g93c790e1.el9 ceph 1.4 M 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-modules-core noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 240 k 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mon x86_64 2:17.2.7-815.g93c790e1.el9 ceph 3.9 M 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-osd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 15 M 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-05T02:40:26.853 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-05T02:40:26.854 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================== 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout:Install 33 Packages 2024-04-05T02:40:26.855 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:26.856 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 27 M 2024-04-05T02:40:26.856 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 96 M 2024-04-05T02:40:26.856 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:40:27.473 INFO:teuthology.orchestra.run.smithi196.stdout:(1/33): ceph-17.2.7-815.g93c790e1.el9.x86_64.rp 29 kB/s | 6.5 kB 00:00 2024-04-05T02:40:27.623 INFO:teuthology.orchestra.run.smithi196.stdout:(2/33): ceph-mgr-17.2.7-815.g93c790e1.el9.x86_6 3.8 MB/s | 1.4 MB 00:00 2024-04-05T02:40:28.039 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:40:28.106 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-05T02:40:28.106 INFO:teuthology.orchestra.run.smithi196.stdout:(3/33): ceph-mds-17.2.7-815.g93c790e1.el9.x86_6 2.4 MB/s | 2.1 MB 00:00 2024-04-05T02:40:28.138 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-05T02:40:28.171 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-05T02:40:28.190 INFO:teuthology.orchestra.run.smithi196.stdout:(4/33): ceph-mgr-modules-core-17.2.7-815.g93c79 2.8 MB/s | 240 kB 00:00 2024-04-05T02:40:28.257 INFO:teuthology.orchestra.run.smithi196.stdout:(5/33): ceph-mon-17.2.7-815.g93c790e1.el9.x86_6 5.0 MB/s | 3.9 MB 00:00 2024-04-05T02:40:28.263 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-05T02:40:28.340 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-05T02:40:28.366 INFO:teuthology.orchestra.run.smithi196.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 372 kB/s | 32 kB 00:00 2024-04-05T02:40:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-05T02:40:28.391 INFO:teuthology.orchestra.run.smithi196.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 719 kB/s | 144 kB 00:00 2024-04-05T02:40:28.408 INFO:teuthology.orchestra.run.smithi196.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 1.1 MB/s | 47 kB 00:00 2024-04-05T02:40:28.425 INFO:teuthology.orchestra.run.smithi196.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 1.1 MB/s | 39 kB 00:00 2024-04-05T02:40:28.454 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-05T02:40:28.508 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-04-05T02:40:28.546 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-04-05T02:40:28.575 INFO:teuthology.orchestra.run.smithi196.stdout:(10/33): ceph-osd-17.2.7-815.g93c790e1.el9.x86_ 16 MB/s | 15 MB 00:00 2024-04-05T02:40:28.595 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-04-05T02:40:28.600 INFO:teuthology.orchestra.run.smithi196.stdout:(11/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.2 MB/s | 241 kB 00:00 2024-04-05T02:40:28.650 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-04-05T02:40:28.659 INFO:teuthology.orchestra.run.smithi196.stdout:(12/33): python3-cryptography-36.0.1-4.el9.x86_ 4.9 MB/s | 1.1 MB 00:00 2024-04-05T02:40:28.685 INFO:teuthology.orchestra.run.smithi196.stdout:(13/33): python3-ply-3.11-14.el9.noarch.rpm 952 kB/s | 103 kB 00:00 2024-04-05T02:40:28.701 INFO:teuthology.orchestra.run.smithi196.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 1.2 MB/s | 124 kB 00:00 2024-04-05T02:40:28.727 INFO:teuthology.orchestra.run.smithi196.stdout:(15/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.7 MB/s | 187 kB 00:00 2024-04-05T02:40:28.855 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-04-05T02:40:28.908 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-04-05T02:40:28.938 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-04-05T02:40:28.973 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-04-05T02:40:28.993 INFO:teuthology.orchestra.run.smithi196.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 428 kB/s | 125 kB 00:00 2024-04-05T02:40:29.003 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-05T02:40:29.071 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-04-05T02:40:29.077 INFO:teuthology.orchestra.run.smithi196.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 116 kB/s | 43 kB 00:00 2024-04-05T02:40:29.105 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:40:29.139 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-04-05T02:40:29.177 INFO:teuthology.orchestra.run.smithi196.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 141 kB/s | 11 kB 00:00 2024-04-05T02:40:29.179 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-04-05T02:40:29.214 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-04-05T02:40:29.219 INFO:teuthology.orchestra.run.smithi196.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 349 kB/s | 172 kB 00:00 2024-04-05T02:40:29.248 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-04-05T02:40:29.261 INFO:teuthology.orchestra.run.smithi196.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 211 kB/s | 18 kB 00:00 2024-04-05T02:40:29.295 INFO:teuthology.orchestra.run.smithi196.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 308 kB/s | 23 kB 00:00 2024-04-05T02:40:29.338 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-04-05T02:40:29.353 INFO:teuthology.orchestra.run.smithi196.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 212 kB/s | 19 kB 00:00 2024-04-05T02:40:29.378 INFO:teuthology.orchestra.run.smithi196.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 235 kB/s | 20 kB 00:00 2024-04-05T02:40:29.379 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-04-05T02:40:29.451 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-04-05T02:40:29.454 INFO:teuthology.orchestra.run.smithi196.stdout:(24/33): python3-cherrypy-18.6.1-2.el9.noarch.r 778 kB/s | 358 kB 00:00 2024-04-05T02:40:29.479 INFO:teuthology.orchestra.run.smithi196.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 368 kB/s | 46 kB 00:00 2024-04-05T02:40:29.505 INFO:teuthology.orchestra.run.smithi196.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 627 kB/s | 79 kB 00:00 2024-04-05T02:40:29.547 INFO:teuthology.orchestra.run.smithi196.stdout:(27/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.9 MB/s | 272 kB 00:00 2024-04-05T02:40:29.572 INFO:teuthology.orchestra.run.smithi196.stdout:(28/33): python3-portend-3.1.0-2.el9.noarch.rpm 178 kB/s | 16 kB 00:00 2024-04-05T02:40:29.596 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-04-05T02:40:29.597 INFO:teuthology.orchestra.run.smithi196.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 978 kB/s | 90 kB 00:00 2024-04-05T02:40:29.614 INFO:teuthology.orchestra.run.smithi196.stdout:(30/33): python3-tempora-5.0.0-2.el9.noarch.rpm 534 kB/s | 36 kB 00:00 2024-04-05T02:40:29.681 INFO:teuthology.orchestra.run.smithi196.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 299 kB/s | 20 kB 00:00 2024-04-05T02:40:29.723 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:40:29.740 INFO:teuthology.orchestra.run.smithi196.stdout:(32/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.3 MB/s | 230 kB 00:00 2024-04-05T02:40:29.765 INFO:teuthology.orchestra.run.smithi196.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.5 MB/s | 427 kB 00:00 2024-04-05T02:40:29.771 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:29.772 INFO:teuthology.orchestra.run.smithi196.stdout:Total 9.2 MB/s | 27 MB 00:02 2024-04-05T02:40:29.786 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:40:29.826 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:40:29.826 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:40:29.878 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 28/33 2024-04-05T02:40:29.909 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:40:29.933 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:40:29.934 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:29.934 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-05T02:40:29.934 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:40:29.934 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:40:29.934 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:30.174 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:40:30.174 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:40:31.310 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:40:31.338 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:40:31.339 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:31.339 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-05T02:40:31.339 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:40:31.339 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:40:31.339 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:31.441 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:40:31.493 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-05T02:40:31.525 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-05T02:40:31.557 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-05T02:40:31.649 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-05T02:40:31.712 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:40:31.719 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-05T02:40:31.736 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:40:31.736 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:31.736 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-05T02:40:31.736 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:40:31.736 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:40:31.736 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:31.760 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-05T02:40:31.832 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-05T02:40:31.886 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-04-05T02:40:31.924 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-04-05T02:40:31.945 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:40:31.968 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:40:31.969 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:31.969 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-05T02:40:31.969 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:40:31.969 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:40:31.969 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:31.973 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-04-05T02:40:32.037 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-04-05T02:40:32.042 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:40:32.223 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-04-05T02:40:32.270 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-04-05T02:40:32.302 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-04-05T02:40:32.336 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-04-05T02:40:32.365 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-05T02:40:32.444 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-04-05T02:40:32.476 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:40:32.509 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-04-05T02:40:32.549 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-04-05T02:40:32.584 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-04-05T02:40:32.619 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-04-05T02:40:32.708 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-04-05T02:40:32.741 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-04-05T02:40:32.814 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-04-05T02:40:32.960 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-04-05T02:40:33.083 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:40:33.240 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 28/33 2024-04-05T02:40:33.271 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:40:33.296 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 29/33 2024-04-05T02:40:33.296 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:33.296 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-05T02:40:33.296 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:40:33.296 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-05T02:40:33.296 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:33.799 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 1/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 3/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 4/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 5/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 6/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-04-05T02:40:33.800 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:40:33.801 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-05T02:40:33.802 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-05T02:40:33.802 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-05T02:40:33.802 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-05T02:40:33.802 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-05T02:40:34.529 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-05T02:40:34.529 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:34.529 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-05T02:40:34.530 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-ply-3.11-14.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-05T02:40:34.531 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:34.532 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:34.664 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:40:34.691 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 30/33 2024-04-05T02:40:34.692 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:34.692 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-05T02:40:34.692 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:40:34.692 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-05T02:40:34.692 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:34.875 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-base 2024-04-05T02:40:35.073 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:40:35.097 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 31/33 2024-04-05T02:40:35.097 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:35.097 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-05T02:40:35.097 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:40:35.097 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-05T02:40:35.098 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:35.290 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:40:35.314 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 32/33 2024-04-05T02:40:35.314 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:35.314 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-05T02:40:35.314 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:40:35.314 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-05T02:40:35.315 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:35.379 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:40:35.453 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:50 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:35.551 INFO:teuthology.orchestra.run.smithi150.stdout:Package ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:40:35.601 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:35.602 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:40:35.603 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:35.668 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install cephadm 2024-04-05T02:40:36.228 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:51 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:36.375 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:36.376 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:36.376 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:40:36.376 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout: cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 82 k 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 82 k 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 372 k 2024-04-05T02:40:36.377 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:36.694 INFO:teuthology.orchestra.run.smithi150.stdout:cephadm-17.2.7-815.g93c790e1.el9.noarch.rpm 259 kB/s | 82 kB 00:00 2024-04-05T02:40:36.695 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:36.695 INFO:teuthology.orchestra.run.smithi150.stdout:Total 258 kB/s | 82 kB 00:00 2024-04-05T02:40:36.695 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:40:36.700 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:40:36.700 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:40:36.703 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:40:36.703 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:40:36.778 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 33/33 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-2:17.2.7-815.g93c790e1.el9.x86_64 1/33 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2/33 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 3/33 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 4/33 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 5/33 2024-04-05T02:40:37.052 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.n 6/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-05T02:40:37.053 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-05T02:40:37.054 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-05T02:40:37.055 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-05T02:40:37.199 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:37.307 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:37.667 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:37.916 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-05T02:40:37.916 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:37.916 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:37.916 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:37.917 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mds-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:37.917 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:37.917 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-modules-core-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:37.917 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mon-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:37.917 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-osd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:37.917 INFO:teuthology.orchestra.run.smithi196.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-05T02:40:37.918 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-05T02:40:37.918 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-05T02:40:37.918 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-05T02:40:37.918 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-05T02:40:37.919 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-05T02:40:37.919 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-05T02:40:37.919 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-05T02:40:37.919 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-05T02:40:37.919 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-05T02:40:37.920 INFO:teuthology.orchestra.run.smithi196.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-05T02:40:37.920 INFO:teuthology.orchestra.run.smithi196.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-05T02:40:37.920 INFO:teuthology.orchestra.run.smithi196.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-05T02:40:37.920 INFO:teuthology.orchestra.run.smithi196.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-05T02:40:37.920 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-05T02:40:37.920 INFO:teuthology.orchestra.run.smithi196.stdout: python3-ply-3.11-14.el9.noarch 2024-04-05T02:40:37.921 INFO:teuthology.orchestra.run.smithi196.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-05T02:40:37.921 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-05T02:40:37.921 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-05T02:40:37.921 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-05T02:40:37.921 INFO:teuthology.orchestra.run.smithi196.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-05T02:40:37.921 INFO:teuthology.orchestra.run.smithi196.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-05T02:40:37.922 INFO:teuthology.orchestra.run.smithi196.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-05T02:40:37.922 INFO:teuthology.orchestra.run.smithi196.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-05T02:40:37.922 INFO:teuthology.orchestra.run.smithi196.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-05T02:40:37.922 INFO:teuthology.orchestra.run.smithi196.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-05T02:40:37.922 INFO:teuthology.orchestra.run.smithi196.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-05T02:40:37.922 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:37.923 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:37.966 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:37.966 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:37.966 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:37.966 INFO:teuthology.orchestra.run.smithi150.stdout: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:37.967 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:37.967 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:38.158 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-immutable-object-cache 2024-04-05T02:40:38.290 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-base 2024-04-05T02:40:38.736 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:53 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:38.862 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:51 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:38.891 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:38.891 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:38.891 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-815.g93c790e1.el9 ceph 137 k 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:40:38.892 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:38.893 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 137 k 2024-04-05T02:40:38.893 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 379 k 2024-04-05T02:40:38.893 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:38.958 INFO:teuthology.orchestra.run.smithi196.stdout:Package ceph-base-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:40:39.009 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:39.010 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:40:39.010 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:39.068 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install cephadm 2024-04-05T02:40:39.092 INFO:teuthology.orchestra.run.smithi150.stdout:ceph-immutable-object-cache-17.2.7-815.g93c790e 690 kB/s | 137 kB 00:00 2024-04-05T02:40:39.093 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:39.093 INFO:teuthology.orchestra.run.smithi150.stdout:Total 684 kB/s | 137 kB 00:00 2024-04-05T02:40:39.093 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:40:39.101 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:40:39.101 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:40:39.152 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:40:39.152 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:40:39.287 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:40:39.352 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:40:39.376 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:40:39.376 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:39.377 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-05T02:40:39.377 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:39.626 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:52 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout: cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 82 k 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:39.775 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:40:39.776 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:39.776 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 82 k 2024-04-05T02:40:39.776 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 372 k 2024-04-05T02:40:39.776 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:40:40.091 INFO:teuthology.orchestra.run.smithi196.stdout:cephadm-17.2.7-815.g93c790e1.el9.noarch.rpm 261 kB/s | 82 kB 00:00 2024-04-05T02:40:40.091 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:40.091 INFO:teuthology.orchestra.run.smithi196.stdout:Total 260 kB/s | 82 kB 00:00 2024-04-05T02:40:40.092 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:40:40.097 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:40:40.097 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:40:40.098 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:40:40.099 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:40:40.173 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:40:40.401 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:40:40.401 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:40.401 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:40.401 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:40.401 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:40.401 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:40.621 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr 2024-04-05T02:40:40.621 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:40.720 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:41.108 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:41.202 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:56 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:41.300 INFO:teuthology.orchestra.run.smithi150.stdout:Package ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:40:41.350 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:41.352 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:40:41.352 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:41.387 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/1 2024-04-05T02:40:41.387 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:41.387 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:41.387 INFO:teuthology.orchestra.run.smithi196.stdout: cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:41.387 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:41.387 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:41.405 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-dashboard 2024-04-05T02:40:41.570 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-immutable-object-cache 2024-04-05T02:40:41.971 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:00:56 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:42.123 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-dashboard noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 1.7 M 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-grafana-dashboards noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 24 k 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-prometheus-alerts noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 15 k 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-05T02:40:42.124 INFO:teuthology.orchestra.run.smithi150.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout:Installing weak dependencies: 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout:Install 7 Packages 2024-04-05T02:40:42.125 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:42.126 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 2.0 M 2024-04-05T02:40:42.126 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 8.1 M 2024-04-05T02:40:42.126 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:42.147 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:55 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:42.297 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:42.297 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:42.297 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repo Size 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-815.g93c790e1.el9 ceph 137 k 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 137 k 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 379 k 2024-04-05T02:40:42.298 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:40:42.530 INFO:teuthology.orchestra.run.smithi196.stdout:ceph-immutable-object-cache-17.2.7-815.g93c790e 592 kB/s | 137 kB 00:00 2024-04-05T02:40:42.531 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:42.531 INFO:teuthology.orchestra.run.smithi196.stdout:Total 589 kB/s | 137 kB 00:00 2024-04-05T02:40:42.531 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:40:42.539 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:40:42.539 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:40:42.590 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:40:42.591 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:40:42.687 INFO:teuthology.orchestra.run.smithi150.stdout:(1/7): ceph-grafana-dashboards-17.2.7-815.g93c7 121 kB/s | 24 kB 00:00 2024-04-05T02:40:42.713 INFO:teuthology.orchestra.run.smithi150.stdout:(2/7): ceph-prometheus-alerts-17.2.7-815.g93c79 66 kB/s | 15 kB 00:00 2024-04-05T02:40:42.748 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:40:42.814 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:40:42.839 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:40:42.840 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:40:42.840 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-05T02:40:42.840 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:42.880 INFO:teuthology.orchestra.run.smithi150.stdout:(3/7): ceph-mgr-dashboard-17.2.7-815.g93c790e1. 4.4 MB/s | 1.7 MB 00:00 2024-04-05T02:40:43.005 INFO:teuthology.orchestra.run.smithi150.stdout:(4/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 28 kB/s | 9.0 kB 00:00 2024-04-05T02:40:43.088 INFO:teuthology.orchestra.run.smithi150.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 109 kB/s | 41 kB 00:00 2024-04-05T02:40:43.213 INFO:teuthology.orchestra.run.smithi150.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 92 kB/s | 31 kB 00:00 2024-04-05T02:40:43.339 INFO:teuthology.orchestra.run.smithi150.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 565 kB/s | 188 kB 00:00 2024-04-05T02:40:43.343 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:43.343 INFO:teuthology.orchestra.run.smithi150.stdout:Total 1.6 MB/s | 2.0 MB 00:01 2024-04-05T02:40:43.344 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:40:43.351 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:40:43.352 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:40:43.416 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:40:43.416 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:40:43.731 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:40:43.761 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.e 1/1 2024-04-05T02:40:43.762 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:43.762 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:43.762 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-immutable-object-cache-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:40:43.762 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:43.762 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:43.799 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-05T02:40:43.832 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-05T02:40:43.878 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-05T02:40:43.919 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-05T02:40:43.948 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 5/7 2024-04-05T02:40:43.988 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-mgr 2024-04-05T02:40:44.177 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 6/7 2024-04-05T02:40:44.218 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:40:44.566 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:57 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:44.663 INFO:teuthology.orchestra.run.smithi196.stdout:Package ceph-mgr-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:40:44.714 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:44.715 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:40:44.715 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:44.730 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:40:44.730 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 1/7 2024-04-05T02:40:44.731 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2/7 2024-04-05T02:40:44.731 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 3/7 2024-04-05T02:40:44.731 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-05T02:40:44.731 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-05T02:40:44.731 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-05T02:40:44.779 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-mgr-dashboard 2024-04-05T02:40:45.030 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:45.031 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:45.249 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-05T02:40:45.343 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:00:58 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:45.494 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:45.494 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-dashboard noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 1.7 M 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-grafana-dashboards noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 24 k 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-prometheus-alerts noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 15 k 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout:Installing weak dependencies: 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-05T02:40:45.495 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout:Install 7 Packages 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 2.0 M 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 8.1 M 2024-04-05T02:40:45.496 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:40:45.759 INFO:teuthology.orchestra.run.smithi196.stdout:(1/7): ceph-grafana-dashboards-17.2.7-815.g93c7 126 kB/s | 24 kB 00:00 2024-04-05T02:40:45.784 INFO:teuthology.orchestra.run.smithi196.stdout:(2/7): ceph-prometheus-alerts-17.2.7-815.g93c79 68 kB/s | 15 kB 00:00 2024-04-05T02:40:45.813 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:00 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:45.927 INFO:teuthology.orchestra.run.smithi196.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 54 kB/s | 9.0 kB 00:00 2024-04-05T02:40:45.952 INFO:teuthology.orchestra.run.smithi196.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 245 kB/s | 41 kB 00:00 2024-04-05T02:40:45.965 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:45.966 INFO:teuthology.orchestra.run.smithi150.stdout:=========================================================================================== 2024-04-05T02:40:45.966 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:40:45.966 INFO:teuthology.orchestra.run.smithi150.stdout:=========================================================================================== 2024-04-05T02:40:45.966 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:45.966 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 7.4 M 2024-04-05T02:40:45.966 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-05T02:40:45.967 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout:=========================================================================================== 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout:Install 12 Packages 2024-04-05T02:40:45.968 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:45.969 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 37 M 2024-04-05T02:40:45.969 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 209 M 2024-04-05T02:40:45.969 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:46.102 INFO:teuthology.orchestra.run.smithi196.stdout:(5/7): ceph-mgr-dashboard-17.2.7-815.g93c790e1. 3.2 MB/s | 1.7 MB 00:00 2024-04-05T02:40:46.128 INFO:teuthology.orchestra.run.smithi196.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 153 kB/s | 31 kB 00:00 2024-04-05T02:40:46.346 INFO:teuthology.orchestra.run.smithi150.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 242 kB/s | 32 kB 00:00 2024-04-05T02:40:46.379 INFO:teuthology.orchestra.run.smithi150.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 534 kB/s | 17 kB 00:00 2024-04-05T02:40:46.421 INFO:teuthology.orchestra.run.smithi150.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 852 kB/s | 34 kB 00:00 2024-04-05T02:40:46.463 INFO:teuthology.orchestra.run.smithi150.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 12 MB/s | 3.0 MB 00:00 2024-04-05T02:40:46.638 INFO:teuthology.orchestra.run.smithi150.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 22 MB/s | 4.7 MB 00:00 2024-04-05T02:40:46.677 INFO:teuthology.orchestra.run.smithi196.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 269 kB/s | 188 kB 00:00 2024-04-05T02:40:46.680 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:46.680 INFO:teuthology.orchestra.run.smithi196.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2024-04-05T02:40:46.680 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:40:46.688 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:40:46.688 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:40:46.705 INFO:teuthology.orchestra.run.smithi150.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 20 MB/s | 4.8 MB 00:00 2024-04-05T02:40:46.731 INFO:teuthology.orchestra.run.smithi150.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.4 MB/s | 217 kB 00:00 2024-04-05T02:40:46.748 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:40:46.748 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:40:46.922 INFO:teuthology.orchestra.run.smithi150.stdout:(8/12): ceph-mgr-diskprediction-local-17.2.7-81 10 MB/s | 7.4 MB 00:00 2024-04-05T02:40:47.039 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:40:47.098 INFO:teuthology.orchestra.run.smithi150.stdout:(9/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 40 MB/s | 16 MB 00:00 2024-04-05T02:40:47.104 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-05T02:40:47.137 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-05T02:40:47.183 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-05T02:40:47.216 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-05T02:40:47.248 INFO:teuthology.orchestra.run.smithi150.stdout:(10/12): libquadmath-11.4.1-3.el9.x86_64.rpm 605 kB/s | 195 kB 00:00 2024-04-05T02:40:47.253 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 5/7 2024-04-05T02:40:47.281 INFO:teuthology.orchestra.run.smithi150.stdout:(11/12): python3-devel-3.9.17-2.el9.x86_64.rpm 1.3 MB/s | 245 kB 00:00 2024-04-05T02:40:47.440 INFO:teuthology.orchestra.run.smithi150.stdout:(12/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.1 MB/s | 803 kB 00:00 2024-04-05T02:40:47.442 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:47.443 INFO:teuthology.orchestra.run.smithi150.stdout:Total 25 MB/s | 37 MB 00:01 2024-04-05T02:40:47.480 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 6/7 2024-04-05T02:40:47.484 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:40:47.509 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:40:47.509 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:40:47.514 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:40:47.952 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:40:47.952 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:40:48.019 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 7/7 2024-04-05T02:40:48.020 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.n 1/7 2024-04-05T02:40:48.020 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2/7 2024-04-05T02:40:48.020 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.no 3/7 2024-04-05T02:40:48.020 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-05T02:40:48.020 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-05T02:40:48.020 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-05T02:40:48.315 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-05T02:40:48.315 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:48.315 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-grafana-dashboards-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-dashboard-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-prometheus-alerts-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-05T02:40:48.316 INFO:teuthology.orchestra.run.smithi196.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-05T02:40:48.317 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:48.317 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:48.546 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-05T02:40:48.676 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:40:48.738 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-04-05T02:40:48.769 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-05T02:40:48.863 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:40:48.906 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-05T02:40:49.085 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:40:49.105 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:02 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:40:49.113 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:40:49.195 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-05T02:40:49.256 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:40:49.257 INFO:teuthology.orchestra.run.smithi196.stdout:=========================================================================================== 2024-04-05T02:40:49.257 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout:=========================================================================================== 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 7.4 M 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-04-05T02:40:49.258 INFO:teuthology.orchestra.run.smithi196.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout:=========================================================================================== 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout:Install 12 Packages 2024-04-05T02:40:49.259 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:49.260 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 37 M 2024-04-05T02:40:49.260 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 209 M 2024-04-05T02:40:49.260 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:40:49.667 INFO:teuthology.orchestra.run.smithi196.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 239 kB/s | 32 kB 00:00 2024-04-05T02:40:49.701 INFO:teuthology.orchestra.run.smithi196.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 532 kB/s | 17 kB 00:00 2024-04-05T02:40:49.743 INFO:teuthology.orchestra.run.smithi196.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 844 kB/s | 34 kB 00:00 2024-04-05T02:40:49.793 INFO:teuthology.orchestra.run.smithi196.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 12 MB/s | 3.0 MB 00:00 2024-04-05T02:40:49.876 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-05T02:40:49.984 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-05T02:40:50.018 INFO:teuthology.orchestra.run.smithi196.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 17 MB/s | 4.7 MB 00:00 2024-04-05T02:40:50.077 INFO:teuthology.orchestra.run.smithi196.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 17 MB/s | 4.8 MB 00:00 2024-04-05T02:40:50.102 INFO:teuthology.orchestra.run.smithi196.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.6 MB/s | 217 kB 00:00 2024-04-05T02:40:50.461 INFO:teuthology.orchestra.run.smithi196.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 41 MB/s | 16 MB 00:00 2024-04-05T02:40:50.528 INFO:teuthology.orchestra.run.smithi196.stdout:(9/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.8 MB/s | 803 kB 00:00 2024-04-05T02:40:50.695 INFO:teuthology.orchestra.run.smithi196.stdout:(10/12): libquadmath-11.4.1-3.el9.x86_64.rpm 837 kB/s | 195 kB 00:00 2024-04-05T02:40:51.187 INFO:teuthology.orchestra.run.smithi196.stdout:(11/12): ceph-mgr-diskprediction-local-17.2.7-8 4.5 MB/s | 7.4 MB 00:01 2024-04-05T02:40:51.233 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-05T02:40:51.495 INFO:teuthology.orchestra.run.smithi196.stdout:(12/12): python3-devel-3.9.17-2.el9.x86_64.rpm 260 kB/s | 245 kB 00:00 2024-04-05T02:40:51.500 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:40:51.501 INFO:teuthology.orchestra.run.smithi196.stdout:Total 17 MB/s | 37 MB 00:02 2024-04-05T02:40:51.542 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:40:51.567 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:40:51.567 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:40:51.976 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:40:51.977 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:40:52.291 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-05T02:40:52.313 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:40:52.678 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:40:52.751 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-04-05T02:40:52.783 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-05T02:40:52.876 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:40:52.928 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-05T02:40:53.105 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:40:53.135 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:40:53.216 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-05T02:40:53.880 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-05T02:40:54.007 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-05T02:40:54.886 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:40:54.886 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 1/12 2024-04-05T02:40:54.886 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-04-05T02:40:54.886 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:40:54.886 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-04-05T02:40:54.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-04-05T02:40:55.264 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-05T02:40:55.293 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-04-05T02:40:55.293 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:55.293 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:40:55.293 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:55.293 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:55.294 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:40:55.716 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-rook 2024-04-05T02:40:56.297 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:11 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:40:56.297 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-05T02:40:56.326 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:40:56.450 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:40:56.451 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================= 2024-04-05T02:40:56.451 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:40:56.451 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================= 2024-04-05T02:40:56.451 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:40:56.451 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-rook noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 47 k 2024-04-05T02:40:56.451 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-05T02:40:56.452 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:56.453 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:40:56.453 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================= 2024-04-05T02:40:56.453 INFO:teuthology.orchestra.run.smithi150.stdout:Install 13 Packages 2024-04-05T02:40:56.453 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:40:56.454 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 2.0 M 2024-04-05T02:40:56.454 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 26 M 2024-04-05T02:40:56.454 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:40:56.737 INFO:teuthology.orchestra.run.smithi150.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 191 kB/s | 24 kB 00:00 2024-04-05T02:40:56.762 INFO:teuthology.orchestra.run.smithi150.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 118 kB/s | 18 kB 00:00 2024-04-05T02:40:56.804 INFO:teuthology.orchestra.run.smithi150.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.6 MB/s | 174 kB 00:00 2024-04-05T02:40:56.830 INFO:teuthology.orchestra.run.smithi150.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.9 MB/s | 132 kB 00:00 2024-04-05T02:40:56.855 INFO:teuthology.orchestra.run.smithi150.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-04-05T02:40:56.880 INFO:teuthology.orchestra.run.smithi150.stdout:(6/13): ceph-mgr-rook-17.2.7-815.g93c790e1.el9. 175 kB/s | 47 kB 00:00 2024-04-05T02:40:56.905 INFO:teuthology.orchestra.run.smithi150.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 570 kB/s | 43 kB 00:00 2024-04-05T02:40:58.839 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 12/12 2024-04-05T02:40:58.839 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790 1/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-04-05T02:40:58.840 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-04-05T02:40:59.219 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-diskprediction-local-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-05T02:40:59.220 INFO:teuthology.orchestra.run.smithi196.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-05T02:40:59.221 INFO:teuthology.orchestra.run.smithi196.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-05T02:40:59.221 INFO:teuthology.orchestra.run.smithi196.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-05T02:40:59.221 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:40:59.221 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:40:59.608 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-mgr-rook 2024-04-05T02:41:00.184 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:13 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:00.336 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================= 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================= 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-rook noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 47 k 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-05T02:41:00.337 INFO:teuthology.orchestra.run.smithi196.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================= 2024-04-05T02:41:00.338 INFO:teuthology.orchestra.run.smithi196.stdout:Install 13 Packages 2024-04-05T02:41:00.339 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:00.339 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 2.0 M 2024-04-05T02:41:00.339 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 26 M 2024-04-05T02:41:00.339 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:00.660 INFO:teuthology.orchestra.run.smithi196.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 138 kB/s | 18 kB 00:00 2024-04-05T02:41:00.685 INFO:teuthology.orchestra.run.smithi196.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 156 kB/s | 24 kB 00:00 2024-04-05T02:41:00.744 INFO:teuthology.orchestra.run.smithi196.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.1 MB/s | 174 kB 00:00 2024-04-05T02:41:00.769 INFO:teuthology.orchestra.run.smithi196.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 132 kB 00:00 2024-04-05T02:41:00.794 INFO:teuthology.orchestra.run.smithi196.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-04-05T02:41:00.820 INFO:teuthology.orchestra.run.smithi196.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 860 kB/s | 43 kB 00:00 2024-04-05T02:41:00.845 INFO:teuthology.orchestra.run.smithi196.stdout:(7/13): ceph-mgr-rook-17.2.7-815.g93c790e1.el9. 150 kB/s | 47 kB 00:00 2024-04-05T02:41:01.054 INFO:teuthology.orchestra.run.smithi196.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 64 kB/s | 15 kB 00:00 2024-04-05T02:41:01.079 INFO:teuthology.orchestra.run.smithi196.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 113 kB/s | 32 kB 00:00 2024-04-05T02:41:01.279 INFO:teuthology.orchestra.run.smithi196.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 295 kB/s | 59 kB 00:00 2024-04-05T02:41:01.305 INFO:teuthology.orchestra.run.smithi196.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 441 kB/s | 203 kB 00:00 2024-04-05T02:41:01.438 INFO:teuthology.orchestra.run.smithi196.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 566 kB/s | 90 kB 00:00 2024-04-05T02:41:01.480 INFO:teuthology.orchestra.run.smithi196.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 2.4 MB/s | 1.0 MB 00:00 2024-04-05T02:41:01.482 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:01.482 INFO:teuthology.orchestra.run.smithi196.stdout:Total 1.8 MB/s | 2.0 MB 00:01 2024-04-05T02:41:01.482 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:01.496 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:01.496 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:01.585 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:01.586 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:02.097 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:02.257 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-05T02:41:02.314 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-05T02:41:02.369 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-05T02:41:02.398 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-05T02:41:02.440 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-05T02:41:02.532 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-05T02:41:02.641 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-05T02:41:02.690 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-05T02:41:03.405 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-05T02:41:03.456 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-05T02:41:03.489 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-05T02:41:03.545 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-05T02:41:03.568 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:41:04.350 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:41:04.350 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 1/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-05T02:41:04.351 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-05T02:41:04.697 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:04.698 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:04.919 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-mgr-cephadm 2024-04-05T02:41:05.486 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:18 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:05.636 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================= 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================= 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 123 k 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-05T02:41:05.637 INFO:teuthology.orchestra.run.smithi196.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-05T02:41:05.638 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:05.638 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:05.638 INFO:teuthology.orchestra.run.smithi196.stdout:======================================================================================= 2024-04-05T02:41:05.638 INFO:teuthology.orchestra.run.smithi196.stdout:Install 6 Packages 2024-04-05T02:41:05.638 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:05.638 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 6.8 M 2024-04-05T02:41:05.639 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 33 M 2024-04-05T02:41:05.639 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:05.730 INFO:teuthology.orchestra.run.smithi150.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 1.7 kB/s | 15 kB 00:08 2024-04-05T02:41:06.265 INFO:teuthology.orchestra.run.smithi196.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.8 MB/s | 248 kB 00:00 2024-04-05T02:41:06.615 INFO:teuthology.orchestra.run.smithi196.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 12 MB/s | 5.8 MB 00:00 2024-04-05T02:41:06.641 INFO:teuthology.orchestra.run.smithi196.stdout:(3/6): ceph-mgr-cephadm-17.2.7-815.g93c790e1.el 241 kB/s | 123 kB 00:00 2024-04-05T02:41:06.680 INFO:teuthology.orchestra.run.smithi150.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 3.3 kB/s | 32 kB 00:09 2024-04-05T02:41:06.849 INFO:teuthology.orchestra.run.smithi196.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 941 kB/s | 548 kB 00:00 2024-04-05T02:41:06.899 INFO:teuthology.orchestra.run.smithi196.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 203 kB/s | 58 kB 00:00 2024-04-05T02:41:06.925 INFO:teuthology.orchestra.run.smithi196.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 180 kB/s | 51 kB 00:00 2024-04-05T02:41:06.927 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:06.928 INFO:teuthology.orchestra.run.smithi196.stdout:Total 5.3 MB/s | 6.8 MB 00:01 2024-04-05T02:41:06.940 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:06.951 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:06.951 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:07.039 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:07.040 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:07.353 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:07.533 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-05T02:41:07.584 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-05T02:41:08.101 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-05T02:41:08.179 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-05T02:41:08.242 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-05T02:41:08.272 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:41:09.076 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:41:09.076 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/6 2024-04-05T02:41:09.076 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-05T02:41:09.076 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-05T02:41:09.076 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-05T02:41:09.076 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:09.371 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:09.625 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install ceph-fuse 2024-04-05T02:41:10.047 INFO:teuthology.orchestra.run.smithi150.stdout:(10/13): python3-google-auth-2.28.2-1.el9.noarc 15 kB/s | 203 kB 00:13 2024-04-05T02:41:10.187 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:23 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:10.336 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repository Size 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 812 k 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:Installing dependencies: 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout:Install 2 Packages 2024-04-05T02:41:10.337 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:10.338 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 891 k 2024-04-05T02:41:10.338 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 2.6 M 2024-04-05T02:41:10.338 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:10.452 INFO:teuthology.orchestra.run.smithi196.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 694 kB/s | 79 kB 00:00 2024-04-05T02:41:10.777 INFO:teuthology.orchestra.run.smithi196.stdout:(2/2): ceph-fuse-17.2.7-815.g93c790e1.el9.x86_6 1.8 MB/s | 812 kB 00:00 2024-04-05T02:41:10.778 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:10.779 INFO:teuthology.orchestra.run.smithi196.stdout:Total 2.0 MB/s | 891 kB 00:00 2024-04-05T02:41:10.779 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:10.799 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:10.799 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:10.871 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:10.871 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:11.062 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:11.191 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-05T02:41:11.252 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:41:11.975 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:41:11.975 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/2 2024-04-05T02:41:12.312 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-05T02:41:12.312 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:12.312 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:12.312 INFO:teuthology.orchestra.run.smithi196.stdout: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-05T02:41:12.313 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:12.313 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:12.641 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install librados-devel 2024-04-05T02:41:12.688 INFO:teuthology.orchestra.run.smithi150.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 34 kB/s | 90 kB 00:02 2024-04-05T02:41:13.217 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:26 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:13.369 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:13.370 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:13.370 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repo Size 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout: librados-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 126 k 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 126 k 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 448 k 2024-04-05T02:41:13.371 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:13.675 INFO:teuthology.orchestra.run.smithi196.stdout:librados-devel-17.2.7-815.g93c790e1.el9.x86_64. 416 kB/s | 126 kB 00:00 2024-04-05T02:41:13.676 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:13.676 INFO:teuthology.orchestra.run.smithi196.stdout:Total 414 kB/s | 126 kB 00:00 2024-04-05T02:41:13.676 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:13.681 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:13.682 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:13.737 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:13.737 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:13.888 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:13.995 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:14.423 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:14.725 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:14.725 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:14.726 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:14.726 INFO:teuthology.orchestra.run.smithi196.stdout: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:14.726 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:14.726 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:14.960 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install libcephfs2 2024-04-05T02:41:15.538 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:28 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:15.638 INFO:teuthology.orchestra.run.smithi196.stdout:Package libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:15.694 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:15.694 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:15.695 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:15.757 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install libcephfs-devel 2024-04-05T02:41:16.323 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:29 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:16.474 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repo Size 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout: libcephfs-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 25 k 2024-04-05T02:41:16.475 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 110 k 2024-04-05T02:41:16.476 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:16.712 INFO:teuthology.orchestra.run.smithi196.stdout:libcephfs-devel-17.2.7-815.g93c790e1.el9.x86_64 105 kB/s | 25 kB 00:00 2024-04-05T02:41:16.713 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:16.713 INFO:teuthology.orchestra.run.smithi196.stdout:Total 105 kB/s | 25 kB 00:00 2024-04-05T02:41:16.713 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:16.716 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:16.716 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:16.717 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:16.718 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:16.817 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:16.944 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:17.333 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:17.592 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:17.592 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:17.592 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:17.593 INFO:teuthology.orchestra.run.smithi196.stdout: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:17.593 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:17.593 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:17.785 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install librados2 2024-04-05T02:41:18.343 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:31 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:18.440 INFO:teuthology.orchestra.run.smithi196.stdout:Package librados2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:18.494 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:18.495 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:18.495 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:18.552 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install librbd1 2024-04-05T02:41:19.104 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:32 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:19.201 INFO:teuthology.orchestra.run.smithi196.stdout:Package librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:19.253 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:19.254 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:19.254 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:19.309 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install python3-rados 2024-04-05T02:41:19.870 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:32 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:19.968 INFO:teuthology.orchestra.run.smithi196.stdout:Package python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:20.021 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:20.022 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:20.022 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:20.085 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install python3-rgw 2024-04-05T02:41:20.651 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:33 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:20.749 INFO:teuthology.orchestra.run.smithi196.stdout:Package python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:20.802 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:20.803 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:20.803 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:20.864 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install python3-cephfs 2024-04-05T02:41:21.427 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:34 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:21.527 INFO:teuthology.orchestra.run.smithi196.stdout:Package python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:21.579 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:21.580 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:21.580 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:21.642 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install python3-rbd 2024-04-05T02:41:22.205 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:35 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:22.303 INFO:teuthology.orchestra.run.smithi196.stdout:Package python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:22.355 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:22.356 INFO:teuthology.orchestra.run.smithi196.stdout:Nothing to do. 2024-04-05T02:41:22.356 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:22.419 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install rbd-fuse 2024-04-05T02:41:22.981 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:35 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:23.134 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout: Package Architecture Version Repository Size 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout: rbd-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 82 k 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:23.135 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:23.136 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:23.136 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:41:23.136 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:23.136 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 82 k 2024-04-05T02:41:23.136 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 222 k 2024-04-05T02:41:23.136 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:23.340 INFO:teuthology.orchestra.run.smithi196.stdout:rbd-fuse-17.2.7-815.g93c790e1.el9.x86_64.rpm 401 kB/s | 82 kB 00:00 2024-04-05T02:41:23.340 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:23.340 INFO:teuthology.orchestra.run.smithi196.stdout:Total 399 kB/s | 82 kB 00:00 2024-04-05T02:41:23.341 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:23.347 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:23.347 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:23.401 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:23.402 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:23.541 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:23.639 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:24.011 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:24.288 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:24.306 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:24.306 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:24.307 INFO:teuthology.orchestra.run.smithi196.stdout: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:24.307 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:24.307 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:24.526 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install rbd-mirror 2024-04-05T02:41:25.104 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:38 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:25.256 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:25.256 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:25.256 INFO:teuthology.orchestra.run.smithi196.stdout: Package Arch Version Repo Size 2024-04-05T02:41:25.256 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:25.256 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:25.256 INFO:teuthology.orchestra.run.smithi196.stdout: rbd-mirror x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 2.9 M 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 12 M 2024-04-05T02:41:25.257 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:25.353 INFO:teuthology.orchestra.run.smithi150.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 53 kB/s | 1.0 MB 00:19 2024-04-05T02:41:25.785 INFO:teuthology.orchestra.run.smithi196.stdout:rbd-mirror-17.2.7-815.g93c790e1.el9.x86_64.rpm 5.6 MB/s | 2.9 MB 00:00 2024-04-05T02:41:25.786 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:25.786 INFO:teuthology.orchestra.run.smithi196.stdout:Total 5.6 MB/s | 2.9 MB 00:00 2024-04-05T02:41:25.786 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:25.795 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:25.795 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:25.867 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:25.867 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:26.267 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:26.327 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:26.351 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:26.351 INFO:teuthology.orchestra.run.smithi196.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:41:26.352 INFO:teuthology.orchestra.run.smithi196.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-05T02:41:26.352 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:41:26.352 INFO:teuthology.orchestra.run.smithi196.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:41:26.352 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:27.435 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:27.435 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:27.435 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:27.435 INFO:teuthology.orchestra.run.smithi196.stdout: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:27.436 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:27.436 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:27.665 DEBUG:teuthology.orchestra.run.smithi196:> sudo yum -y install rbd-nbd 2024-04-05T02:41:28.238 INFO:teuthology.orchestra.run.smithi196.stdout:Last metadata expiration check: 0:01:41 ago on Fri 05 Apr 2024 02:39:47 AM UTC. 2024-04-05T02:41:28.388 INFO:teuthology.orchestra.run.smithi196.stdout:Dependencies resolved. 2024-04-05T02:41:28.389 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:28.389 INFO:teuthology.orchestra.run.smithi196.stdout: Package Architecture Version Repository Size 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:Installing: 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout: rbd-nbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 165 k 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction Summary 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:================================================================================ 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:Install 1 Package 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:Total download size: 165 k 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:Installed size: 477 k 2024-04-05T02:41:28.390 INFO:teuthology.orchestra.run.smithi196.stdout:Downloading Packages: 2024-04-05T02:41:28.788 INFO:teuthology.orchestra.run.smithi196.stdout:rbd-nbd-17.2.7-815.g93c790e1.el9.x86_64.rpm 414 kB/s | 165 kB 00:00 2024-04-05T02:41:28.789 INFO:teuthology.orchestra.run.smithi196.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:28.789 INFO:teuthology.orchestra.run.smithi196.stdout:Total 413 kB/s | 165 kB 00:00 2024-04-05T02:41:28.789 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction check 2024-04-05T02:41:28.797 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction check succeeded. 2024-04-05T02:41:28.797 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction test 2024-04-05T02:41:28.854 INFO:teuthology.orchestra.run.smithi196.stdout:Transaction test succeeded. 2024-04-05T02:41:28.854 INFO:teuthology.orchestra.run.smithi196.stdout:Running transaction 2024-04-05T02:41:29.016 INFO:teuthology.orchestra.run.smithi196.stdout: Preparing : 1/1 2024-04-05T02:41:29.114 INFO:teuthology.orchestra.run.smithi196.stdout: Installing : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:29.561 INFO:teuthology.orchestra.run.smithi196.stdout: Running scriptlet: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:29.852 INFO:teuthology.orchestra.run.smithi196.stdout: Verifying : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:29.853 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:29.853 INFO:teuthology.orchestra.run.smithi196.stdout:Installed: 2024-04-05T02:41:29.853 INFO:teuthology.orchestra.run.smithi196.stdout: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:29.853 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:41:29.853 INFO:teuthology.orchestra.run.smithi196.stdout:Complete! 2024-04-05T02:41:30.082 DEBUG:teuthology.parallel:result is None 2024-04-05T02:41:34.394 INFO:teuthology.orchestra.run.smithi150.stdout:(13/13): python3-rsa-4.9-2.el9.noarch.rpm 2.1 kB/s | 59 kB 00:27 2024-04-05T02:41:34.396 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:34.397 INFO:teuthology.orchestra.run.smithi150.stdout:Total 55 kB/s | 2.0 MB 00:37 2024-04-05T02:41:34.397 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:34.412 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:34.412 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:34.508 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:34.511 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:35.030 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:35.200 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-05T02:41:35.262 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-05T02:41:35.317 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-05T02:41:35.355 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-05T02:41:35.388 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-05T02:41:35.482 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-05T02:41:35.582 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-05T02:41:35.639 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-05T02:41:36.362 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-05T02:41:36.420 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-05T02:41:36.453 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-05T02:41:36.500 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-05T02:41:36.532 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:41:37.256 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 13/13 2024-04-05T02:41:37.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 1/13 2024-04-05T02:41:37.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-05T02:41:37.256 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-05T02:41:37.257 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-05T02:41:37.630 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-05T02:41:37.630 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:37.630 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:37.630 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-rook-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:41:37.630 INFO:teuthology.orchestra.run.smithi150.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-05T02:41:37.630 INFO:teuthology.orchestra.run.smithi150.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:37.631 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:37.884 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-mgr-cephadm 2024-04-05T02:41:38.473 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:53 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================= 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================= 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-cephadm noarch 2:17.2.7-815.g93c790e1.el9 ceph-noarch 123 k 2024-04-05T02:41:38.625 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout:======================================================================================= 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout:Install 6 Packages 2024-04-05T02:41:38.626 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:38.627 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 6.8 M 2024-04-05T02:41:38.627 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 33 M 2024-04-05T02:41:38.627 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:41:39.121 INFO:teuthology.orchestra.run.smithi150.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 5.8 MB 00:00 2024-04-05T02:41:39.146 INFO:teuthology.orchestra.run.smithi150.stdout:(2/6): ceph-mgr-cephadm-17.2.7-815.g93c790e1.el 403 kB/s | 123 kB 00:00 2024-04-05T02:41:39.171 INFO:teuthology.orchestra.run.smithi150.stdout:(3/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 749 kB/s | 248 kB 00:00 2024-04-05T02:41:39.472 INFO:teuthology.orchestra.run.smithi150.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 177 kB/s | 58 kB 00:00 2024-04-05T02:41:39.497 INFO:teuthology.orchestra.run.smithi150.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 157 kB/s | 51 kB 00:00 2024-04-05T02:41:39.622 INFO:teuthology.orchestra.run.smithi150.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.1 MB/s | 548 kB 00:00 2024-04-05T02:41:39.625 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:39.625 INFO:teuthology.orchestra.run.smithi150.stdout:Total 6.8 MB/s | 6.8 MB 00:00 2024-04-05T02:41:39.638 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:39.648 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:39.649 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:39.736 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:39.736 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:40.030 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:40.198 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-05T02:41:40.274 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-05T02:41:40.798 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-05T02:41:40.885 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-05T02:41:40.940 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-05T02:41:40.969 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:41:41.769 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 6/6 2024-04-05T02:41:41.769 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 1/6 2024-04-05T02:41:41.769 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-05T02:41:41.769 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-05T02:41:41.769 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-05T02:41:41.770 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-05T02:41:42.065 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-05T02:41:42.065 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:42.065 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-mgr-cephadm-2:17.2.7-815.g93c790e1.el9.noarch 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:42.066 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:42.323 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install ceph-fuse 2024-04-05T02:41:42.890 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:01:57 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:43.041 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:43.041 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:43.041 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repository Size 2024-04-05T02:41:43.041 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:43.041 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:41:43.041 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 812 k 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout:Installing dependencies: 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout:Install 2 Packages 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:43.042 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 891 k 2024-04-05T02:41:43.043 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 2.6 M 2024-04-05T02:41:43.043 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:41:43.157 INFO:teuthology.orchestra.run.smithi150.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 688 kB/s | 79 kB 00:00 2024-04-05T02:41:43.349 INFO:teuthology.orchestra.run.smithi150.stdout:(2/2): ceph-fuse-17.2.7-815.g93c790e1.el9.x86_6 2.6 MB/s | 812 kB 00:00 2024-04-05T02:41:43.350 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:43.351 INFO:teuthology.orchestra.run.smithi150.stdout:Total 2.8 MB/s | 891 kB 00:00 2024-04-05T02:41:43.351 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:43.371 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:43.371 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:43.444 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:43.445 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:43.651 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:43.786 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-05T02:41:43.857 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:41:44.559 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2/2 2024-04-05T02:41:44.560 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/2 2024-04-05T02:41:44.849 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-05T02:41:44.850 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:44.850 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:44.850 INFO:teuthology.orchestra.run.smithi150.stdout: ceph-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-05T02:41:44.850 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:44.850 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:45.126 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librados-devel 2024-04-05T02:41:45.701 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:00 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:45.858 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout: librados-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 126 k 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:41:45.859 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:45.860 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 126 k 2024-04-05T02:41:45.860 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 448 k 2024-04-05T02:41:45.860 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:41:46.087 INFO:teuthology.orchestra.run.smithi150.stdout:librados-devel-17.2.7-815.g93c790e1.el9.x86_64. 557 kB/s | 126 kB 00:00 2024-04-05T02:41:46.087 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:46.090 INFO:teuthology.orchestra.run.smithi150.stdout:Total 553 kB/s | 126 kB 00:00 2024-04-05T02:41:46.090 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:46.093 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:46.093 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:46.148 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:46.149 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:46.305 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:46.411 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:46.878 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:47.156 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:47.156 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:47.156 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:47.156 INFO:teuthology.orchestra.run.smithi150.stdout: librados-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:47.157 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:47.157 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:47.392 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install libcephfs2 2024-04-05T02:41:47.977 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:02 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:48.077 INFO:teuthology.orchestra.run.smithi150.stdout:Package libcephfs2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:48.132 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:48.133 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:48.133 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:48.188 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install libcephfs-devel 2024-04-05T02:41:48.763 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:03 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:48.914 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:48.915 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:48.915 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-04-05T02:41:48.915 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs-devel x86_64 2:17.2.7-815.g93c790e1.el9 ceph 25 k 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 25 k 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 110 k 2024-04-05T02:41:48.916 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:41:49.099 INFO:teuthology.orchestra.run.smithi150.stdout:libcephfs-devel-17.2.7-815.g93c790e1.el9.x86_64 137 kB/s | 25 kB 00:00 2024-04-05T02:41:49.099 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:49.099 INFO:teuthology.orchestra.run.smithi150.stdout:Total 136 kB/s | 25 kB 00:00 2024-04-05T02:41:49.100 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:49.103 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:49.103 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:49.104 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:49.105 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:49.187 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:49.296 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:49.670 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:49.922 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:49.922 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:49.923 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:49.923 INFO:teuthology.orchestra.run.smithi150.stdout: libcephfs-devel-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:49.923 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:49.923 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:50.100 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librados2 2024-04-05T02:41:50.670 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:05 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:50.768 INFO:teuthology.orchestra.run.smithi150.stdout:Package librados2-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:50.820 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:50.821 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:50.821 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:50.879 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install librbd1 2024-04-05T02:41:51.441 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:06 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:51.539 INFO:teuthology.orchestra.run.smithi150.stdout:Package librbd1-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:51.591 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:51.699 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:51.699 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:51.699 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rados 2024-04-05T02:41:52.263 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:07 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:52.362 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rados-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:52.414 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:52.415 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:52.415 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:52.468 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rgw 2024-04-05T02:41:53.036 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:08 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:53.134 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rgw-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:53.187 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:53.188 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:53.188 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:53.243 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-cephfs 2024-04-05T02:41:53.811 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:08 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:53.910 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-cephfs-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:53.963 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:53.964 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:53.964 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:54.024 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install python3-rbd 2024-04-05T02:41:54.593 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:09 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:54.693 INFO:teuthology.orchestra.run.smithi150.stdout:Package python3-rbd-2:17.2.7-815.g93c790e1.el9.x86_64 is already installed. 2024-04-05T02:41:54.746 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:54.746 INFO:teuthology.orchestra.run.smithi150.stdout:Nothing to do. 2024-04-05T02:41:54.747 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:54.806 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-fuse 2024-04-05T02:41:55.373 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:10 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:55.523 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-fuse x86_64 2:17.2.7-815.g93c790e1.el9 ceph 82 k 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:41:55.524 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:55.525 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 82 k 2024-04-05T02:41:55.525 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 222 k 2024-04-05T02:41:55.525 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:41:55.735 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-fuse-17.2.7-815.g93c790e1.el9.x86_64.rpm 390 kB/s | 82 kB 00:00 2024-04-05T02:41:55.735 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:55.735 INFO:teuthology.orchestra.run.smithi150.stdout:Total 388 kB/s | 82 kB 00:00 2024-04-05T02:41:55.736 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:55.743 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:55.743 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:55.799 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:55.799 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:55.946 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:56.059 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:56.448 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:56.732 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:56.732 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:56.732 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:56.732 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-fuse-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:56.732 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:56.732 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:41:56.935 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-mirror 2024-04-05T02:41:57.507 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:12 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:41:57.662 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout: Package Arch Version Repo Size 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-mirror x86_64 2:17.2.7-815.g93c790e1.el9 ceph 2.9 M 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:57.663 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:41:57.664 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:41:57.664 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:41:57.664 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:57.664 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 2.9 M 2024-04-05T02:41:57.664 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 12 M 2024-04-05T02:41:57.664 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:41:58.206 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-mirror-17.2.7-815.g93c790e1.el9.x86_64.rpm 5.4 MB/s | 2.9 MB 00:00 2024-04-05T02:41:58.206 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:41:58.207 INFO:teuthology.orchestra.run.smithi150.stdout:Total 5.4 MB/s | 2.9 MB 00:00 2024-04-05T02:41:58.207 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:41:58.216 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:41:58.216 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:41:58.287 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:41:58.288 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:41:58.693 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:41:58.765 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:58.789 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:58.790 INFO:teuthology.orchestra.run.smithi150.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-05T02:41:58.790 INFO:teuthology.orchestra.run.smithi150.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-05T02:41:58.790 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:41:58.790 INFO:teuthology.orchestra.run.smithi150.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-05T02:41:58.790 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:59.887 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:41:59.888 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:59.888 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:41:59.888 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-mirror-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:41:59.888 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:41:59.888 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:42:00.109 DEBUG:teuthology.orchestra.run.smithi150:> sudo yum -y install rbd-nbd 2024-04-05T02:42:00.688 INFO:teuthology.orchestra.run.smithi150.stdout:Last metadata expiration check: 0:02:15 ago on Fri 05 Apr 2024 02:39:45 AM UTC. 2024-04-05T02:42:00.839 INFO:teuthology.orchestra.run.smithi150.stdout:Dependencies resolved. 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout: Package Architecture Version Repository Size 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout:Installing: 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-nbd x86_64 2:17.2.7-815.g93c790e1.el9 ceph 165 k 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction Summary 2024-04-05T02:42:00.840 INFO:teuthology.orchestra.run.smithi150.stdout:================================================================================ 2024-04-05T02:42:00.841 INFO:teuthology.orchestra.run.smithi150.stdout:Install 1 Package 2024-04-05T02:42:00.841 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:42:00.841 INFO:teuthology.orchestra.run.smithi150.stdout:Total download size: 165 k 2024-04-05T02:42:00.841 INFO:teuthology.orchestra.run.smithi150.stdout:Installed size: 477 k 2024-04-05T02:42:00.841 INFO:teuthology.orchestra.run.smithi150.stdout:Downloading Packages: 2024-04-05T02:42:01.193 INFO:teuthology.orchestra.run.smithi150.stdout:rbd-nbd-17.2.7-815.g93c790e1.el9.x86_64.rpm 470 kB/s | 165 kB 00:00 2024-04-05T02:42:01.193 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:42:01.194 INFO:teuthology.orchestra.run.smithi150.stdout:Total 468 kB/s | 165 kB 00:00 2024-04-05T02:42:01.194 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction check 2024-04-05T02:42:01.201 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction check succeeded. 2024-04-05T02:42:01.202 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction test 2024-04-05T02:42:01.257 INFO:teuthology.orchestra.run.smithi150.stdout:Transaction test succeeded. 2024-04-05T02:42:01.257 INFO:teuthology.orchestra.run.smithi150.stdout:Running transaction 2024-04-05T02:42:01.402 INFO:teuthology.orchestra.run.smithi150.stdout: Preparing : 1/1 2024-04-05T02:42:01.499 INFO:teuthology.orchestra.run.smithi150.stdout: Installing : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:42:01.938 INFO:teuthology.orchestra.run.smithi150.stdout: Running scriptlet: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:42:02.245 INFO:teuthology.orchestra.run.smithi150.stdout: Verifying : rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 1/1 2024-04-05T02:42:02.245 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:42:02.245 INFO:teuthology.orchestra.run.smithi150.stdout:Installed: 2024-04-05T02:42:02.245 INFO:teuthology.orchestra.run.smithi150.stdout: rbd-nbd-2:17.2.7-815.g93c790e1.el9.x86_64 2024-04-05T02:42:02.245 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:42:02.245 INFO:teuthology.orchestra.run.smithi150.stdout:Complete! 2024-04-05T02:42:02.483 DEBUG:teuthology.parallel:result is None 2024-04-05T02:42:02.483 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:42:02.483 INFO:teuthology.packaging:ref: None 2024-04-05T02:42:02.483 INFO:teuthology.packaging:tag: None 2024-04-05T02:42:02.484 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:42:02.484 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:42:02.484 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:42:02.799 DEBUG:teuthology.orchestra.run.smithi150:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-05T02:42:02.820 INFO:teuthology.orchestra.run.smithi150.stdout:17.2.7-815.g93c790e1.el9 2024-04-05T02:42:02.821 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-815.g93c790e1.el9 2024-04-05T02:42:02.821 INFO:teuthology.task.install:The correct ceph version 17.2.7-815.g93c790e1 is installed. 2024-04-05T02:42:02.822 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-05T02:42:02.823 INFO:teuthology.packaging:ref: None 2024-04-05T02:42:02.823 INFO:teuthology.packaging:tag: None 2024-04-05T02:42:02.823 INFO:teuthology.packaging:branch: quincy 2024-04-05T02:42:02.823 INFO:teuthology.packaging:sha1: 0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:42:02.823 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-05T02:42:03.042 DEBUG:teuthology.orchestra.run.smithi196:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-05T02:42:03.067 INFO:teuthology.orchestra.run.smithi196.stdout:17.2.7-815.g93c790e1.el9 2024-04-05T02:42:03.067 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-815.g93c790e1.el9 2024-04-05T02:42:03.067 INFO:teuthology.task.install:The correct ceph version 17.2.7-815.g93c790e1 is installed. 2024-04-05T02:42:03.069 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-05T02:42:03.069 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:42:03.069 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-05T02:42:03.099 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:42:03.099 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-05T02:42:03.138 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-05T02:42:03.138 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:42:03.138 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/daemon-helper 2024-04-05T02:42:03.167 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-05T02:42:03.235 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:42:03.235 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/usr/bin/daemon-helper 2024-04-05T02:42:03.266 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-05T02:42:03.336 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-05T02:42:03.336 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:42:03.337 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-05T02:42:03.364 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-05T02:42:03.432 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:42:03.432 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-05T02:42:03.460 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-05T02:42:03.528 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-05T02:42:03.528 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:42:03.528 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/usr/bin/stdin-killer 2024-04-05T02:42:03.555 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-05T02:42:03.622 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:42:03.622 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/usr/bin/stdin-killer 2024-04-05T02:42:03.649 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-05T02:42:03.717 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-05T02:42:03.819 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '0028525c15e19309c20289c609e4fb9d0b12dd06'} 2024-04-05T02:42:03.820 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-05T02:42:03.820 INFO:tasks.cephadm:Cluster fsid is 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:03.820 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-05T02:42:03.820 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.150', 'mon.c': '[v2:172.21.15.150:3301,v1:172.21.15.150:6790]', 'mon.b': '172.21.15.196'} 2024-04-05T02:42:03.820 INFO:tasks.cephadm:First mon is mon.a on smithi150 2024-04-05T02:42:03.821 INFO:tasks.cephadm:First mgr is y 2024-04-05T02:42:03.821 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-05T02:42:03.821 DEBUG:teuthology.orchestra.run.smithi150:> sudo hostname $(hostname -s) 2024-04-05T02:42:03.848 DEBUG:teuthology.orchestra.run.smithi196:> sudo hostname $(hostname -s) 2024-04-05T02:42:03.876 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-05T02:42:03.877 DEBUG:teuthology.orchestra.run.smithi150:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-05T02:42:04.114 INFO:teuthology.orchestra.run.smithi150.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 5 02:42 /home/ubuntu/cephtest/cephadm 2024-04-05T02:42:04.114 DEBUG:teuthology.orchestra.run.smithi196:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-05T02:42:04.297 INFO:teuthology.orchestra.run.smithi196.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 5 02:42 /home/ubuntu/cephtest/cephadm 2024-04-05T02:42:04.298 DEBUG:teuthology.orchestra.run.smithi150:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-05T02:42:04.314 DEBUG:teuthology.orchestra.run.smithi196:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-05T02:42:04.340 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-05T02:42:04.340 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-05T02:42:04.356 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-05T02:42:05.185 INFO:teuthology.orchestra.run.smithi196.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:42:05.303 INFO:teuthology.orchestra.run.smithi150.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:42:30.913 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2024-04-05T02:42:30.914 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph_version": "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)", 2024-04-05T02:42:30.914 INFO:teuthology.orchestra.run.smithi150.stdout: "image_id": "a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48", 2024-04-05T02:42:30.914 INFO:teuthology.orchestra.run.smithi150.stdout: "repo_digests": [ 2024-04-05T02:42:30.914 INFO:teuthology.orchestra.run.smithi150.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030" 2024-04-05T02:42:30.914 INFO:teuthology.orchestra.run.smithi150.stdout: ] 2024-04-05T02:42:30.914 INFO:teuthology.orchestra.run.smithi150.stdout:} 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout:{ 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout: "ceph_version": "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)", 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout: "image_id": "a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48", 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout: "repo_digests": [ 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030" 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout: ] 2024-04-05T02:42:35.387 INFO:teuthology.orchestra.run.smithi196.stdout:} 2024-04-05T02:42:35.406 DEBUG:teuthology.orchestra.run.smithi150:> sudo mkdir -p /etc/ceph 2024-04-05T02:42:35.441 DEBUG:teuthology.orchestra.run.smithi196:> sudo mkdir -p /etc/ceph 2024-04-05T02:42:35.483 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 777 /etc/ceph 2024-04-05T02:42:35.513 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod 777 /etc/ceph 2024-04-05T02:42:35.550 INFO:tasks.cephadm:Writing seed config... 2024-04-05T02:42:35.551 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-05T02:42:35.551 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-05T02:42:35.551 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-05T02:42:35.551 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-05T02:42:35.552 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-05T02:42:35.552 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:42:35.553 DEBUG:teuthology.orchestra.run.smithi150:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-05T02:42:35.569 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 15e4009a-f2f6-11ee-b649-cb9ed24678a4 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-05T02:42:35.569 DEBUG:teuthology.orchestra.run.smithi150:mon.a> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service 2024-04-05T02:42:35.611 DEBUG:teuthology.orchestra.run.smithi150:mgr.y> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service 2024-04-05T02:42:35.654 INFO:tasks.cephadm:Bootstrapping... 2024-04-05T02:42:35.655 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.150 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-05T02:42:35.861 INFO:teuthology.orchestra.run.smithi150.stdout:-------------------------------------------------------------------------------- 2024-04-05T02:42:35.861 INFO:teuthology.orchestra.run.smithi150.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '15e4009a-f2f6-11ee-b649-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.150', '--skip-admin-label'] 2024-04-05T02:42:35.887 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-05T02:42:35.887 INFO:teuthology.orchestra.run.smithi150.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-05T02:42:35.887 INFO:teuthology.orchestra.run.smithi150.stdout:Verifying podman|docker is present... 2024-04-05T02:42:35.911 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-05T02:42:35.911 INFO:teuthology.orchestra.run.smithi150.stdout:Verifying lvm2 is present... 2024-04-05T02:42:35.911 INFO:teuthology.orchestra.run.smithi150.stdout:Verifying time synchronization is in place... 2024-04-05T02:42:35.919 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-05T02:42:35.919 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-05T02:42:35.925 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-05T02:42:35.925 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout inactive 2024-04-05T02:42:35.932 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout enabled 2024-04-05T02:42:35.938 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout active 2024-04-05T02:42:35.938 INFO:teuthology.orchestra.run.smithi150.stdout:Unit chronyd.service is enabled and running 2024-04-05T02:42:35.938 INFO:teuthology.orchestra.run.smithi150.stdout:Repeating the final host check... 2024-04-05T02:42:35.962 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-05T02:42:35.962 INFO:teuthology.orchestra.run.smithi150.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-05T02:42:35.962 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl is present 2024-04-05T02:42:35.962 INFO:teuthology.orchestra.run.smithi150.stdout:lvcreate is present 2024-04-05T02:42:35.969 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-05T02:42:35.969 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-05T02:42:35.975 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-05T02:42:35.975 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout inactive 2024-04-05T02:42:35.981 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout enabled 2024-04-05T02:42:35.987 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout active 2024-04-05T02:42:35.988 INFO:teuthology.orchestra.run.smithi150.stdout:Unit chronyd.service is enabled and running 2024-04-05T02:42:35.988 INFO:teuthology.orchestra.run.smithi150.stdout:Host looks OK 2024-04-05T02:42:35.988 INFO:teuthology.orchestra.run.smithi150.stdout:Cluster fsid: 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:35.988 INFO:teuthology.orchestra.run.smithi150.stdout:Acquiring lock 140373821367392 on /run/cephadm/15e4009a-f2f6-11ee-b649-cb9ed24678a4.lock 2024-04-05T02:42:35.988 INFO:teuthology.orchestra.run.smithi150.stdout:Lock 140373821367392 acquired on /run/cephadm/15e4009a-f2f6-11ee-b649-cb9ed24678a4.lock 2024-04-05T02:42:35.988 INFO:teuthology.orchestra.run.smithi150.stdout:Verifying IP 172.21.15.150 port 3300 ... 2024-04-05T02:42:35.989 INFO:teuthology.orchestra.run.smithi150.stdout:Verifying IP 172.21.15.150 port 6789 ... 2024-04-05T02:42:35.989 INFO:teuthology.orchestra.run.smithi150.stdout:Base mon IP(s) is [172.21.15.150:3300, 172.21.15.150:6789], mon addrv is [v2:172.21.15.150:3300,v1:172.21.15.150:6789] 2024-04-05T02:42:35.992 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.150 metric 100 2024-04-05T02:42:35.992 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.150 metric 100 2024-04-05T02:42:35.995 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-05T02:42:35.995 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-05T02:42:35.995 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1798sec hoplimit 64 pref medium 2024-04-05T02:42:35.997 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-05T02:42:35.997 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-05T02:42:35.998 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-05T02:42:35.998 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-05T02:42:35.998 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:6e35/64 scope link 2024-04-05T02:42:35.998 INFO:teuthology.orchestra.run.smithi150.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-05T02:42:35.999 INFO:teuthology.orchestra.run.smithi150.stdout:Mon IP `172.21.15.150` is in CIDR network `172.21.0.0/20` 2024-04-05T02:42:35.999 INFO:teuthology.orchestra.run.smithi150.stdout:Mon IP `172.21.15.150` is in CIDR network `172.21.0.0/20` 2024-04-05T02:42:35.999 INFO:teuthology.orchestra.run.smithi150.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-05T02:42:35.999 INFO:teuthology.orchestra.run.smithi150.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-05T02:42:35.999 INFO:teuthology.orchestra.run.smithi150.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:42:36.463 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stdout a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 2024-04-05T02:42:36.463 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-05T02:42:36.463 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stderr Getting image source signatures 2024-04-05T02:42:36.463 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stderr Copying blob sha256:77070339c81bcb5c53bbbf3f31a47f7187e5393cb0e096452852a31192d97570 2024-04-05T02:42:36.463 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stderr Copying blob sha256:7c8e66990dd5fdc46cc6d5e1ff7927cb8fbc9e77ba63bb5152631196a3d52311 2024-04-05T02:42:36.463 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stderr Copying config sha256:a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 2024-04-05T02:42:36.464 INFO:teuthology.orchestra.run.smithi150.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-05T02:42:37.485 INFO:teuthology.orchestra.run.smithi150.stdout:ceph: stdout ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable) 2024-04-05T02:42:37.485 INFO:teuthology.orchestra.run.smithi150.stdout:Ceph version: ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable) 2024-04-05T02:42:37.486 INFO:teuthology.orchestra.run.smithi150.stdout:Extracting ceph user uid/gid from container image... 2024-04-05T02:42:38.373 INFO:teuthology.orchestra.run.smithi150.stdout:stat: stdout 167 167 2024-04-05T02:42:38.373 INFO:teuthology.orchestra.run.smithi150.stdout:Creating initial keys... 2024-04-05T02:42:39.388 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph-authtool: stdout AQAeZQ9mDr+1LBAAzKRSyXWwzbKOxE/ewHvkew== 2024-04-05T02:42:40.334 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph-authtool: stdout AQAfZQ9mpCgOKhAA7wc2DIdwCV6DKbIOp1K4UA== 2024-04-05T02:42:41.314 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph-authtool: stdout AQAgZQ9mEfJjJxAAs8YfnuFg/Wno5pJGzFzE5g== 2024-04-05T02:42:41.315 INFO:teuthology.orchestra.run.smithi150.stdout:Creating initial monmap... 2024-04-05T02:42:42.396 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-05T02:42:42.396 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-05T02:42:42.396 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:42.396 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-05T02:42:42.397 INFO:teuthology.orchestra.run.smithi150.stdout:monmaptool for a [v2:172.21.15.150:3300,v1:172.21.15.150:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-05T02:42:42.397 INFO:teuthology.orchestra.run.smithi150.stdout:setting min_mon_release = octopus 2024-04-05T02:42:42.397 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/monmaptool: set fsid to 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:42.397 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-05T02:42:42.397 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:42:42.397 INFO:teuthology.orchestra.run.smithi150.stdout:Creating mon... 2024-04-05T02:42:48.090 INFO:teuthology.orchestra.run.smithi150.stdout:create mon.a on 2024-04-05T02:42:48.487 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-05T02:42:48.948 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-05T02:42:49.367 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4.target. 2024-04-05T02:42:49.368 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4.target → /etc/systemd/system/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4.target. 2024-04-05T02:42:49.703 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a 2024-04-05T02:42:49.703 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Failed to reset failed state of unit ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service: Unit ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service not loaded. 2024-04-05T02:42:50.040 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4.target.wants/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service → /etc/systemd/system/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@.service. 2024-04-05T02:42:50.414 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 podman[29047]: 2024-04-05 02:42:50.185670242 +0000 UTC m=+0.017890001 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-05T02:42:50.414 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 podman[29047]: 2024-04-05 02:42:50.415044973 +0000 UTC m=+0.247264715 container create 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-04-05T02:42:50.694 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 podman[29047]: 2024-04-05 02:42:50.650860255 +0000 UTC m=+0.483080001 container init 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240403) 2024-04-05T02:42:50.694 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 podman[29047]: 2024-04-05 02:42:50.656357305 +0000 UTC m=+0.488577047 container start 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-04-05T02:42:50.755 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:42:50.755 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout disabled 2024-04-05T02:42:50.762 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:42:50.762 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout inactive 2024-04-05T02:42:50.762 INFO:teuthology.orchestra.run.smithi150.stdout:firewalld.service is not enabled 2024-04-05T02:42:50.762 INFO:teuthology.orchestra.run.smithi150.stdout:Not possible to enable service . firewalld.service is not available 2024-04-05T02:42:50.763 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for mon to start... 2024-04-05T02:42:50.763 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for mon... 2024-04-05T02:42:50.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 ceph-mon[29085]: mkfs 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:50.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 ceph-mon[29085]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-05T02:42:50.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 bash[29047]: 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 2024-04-05T02:42:50.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:50 smithi150 systemd[1]: Started Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:42:51.866 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:51 smithi150 ceph-mon[29085]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-05T02:42:51.866 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:51 smithi150 ceph-mon[29085]: monmap e1: 1 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:42:51.866 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:51 smithi150 ceph-mon[29085]: fsmap 2024-04-05T02:42:51.867 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:51 smithi150 ceph-mon[29085]: osdmap e1: 0 total, 0 up, 0 in 2024-04-05T02:42:51.867 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:51 smithi150 ceph-mon[29085]: mgrmap e1: no daemons active 2024-04-05T02:42:51.867 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:51 smithi150 ceph-mon[29085]: from='client.? 172.21.15.150:0/549025340' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-05T02:42:51.869 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout cluster: 2024-04-05T02:42:51.870 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout id: 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout services: 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.615046s) 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout data: 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout pgs: 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:42:51.871 INFO:teuthology.orchestra.run.smithi150.stdout:mon is available 2024-04-05T02:42:51.872 INFO:teuthology.orchestra.run.smithi150.stdout:Assimilating anything we can from ceph.conf... 2024-04-05T02:42:52.892 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:42:52.892 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout [global] 2024-04-05T02:42:52.892 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout fsid = 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.150:3300,v1:172.21.15.150:6789] 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout [osd] 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-05T02:42:52.893 INFO:teuthology.orchestra.run.smithi150.stdout:Generating new minimal ceph.conf... 2024-04-05T02:42:52.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:52 smithi150 ceph-mon[29085]: from='client.? 172.21.15.150:0/20946556' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-05T02:42:52.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:52 smithi150 ceph-mon[29085]: from='client.? 172.21.15.150:0/20946556' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-05T02:42:53.962 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:53 smithi150 ceph-mon[29085]: from='client.? 172.21.15.150:0/658891492' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:42:53.965 INFO:teuthology.orchestra.run.smithi150.stdout:Restarting the monitor... 2024-04-05T02:42:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:53 smithi150 systemd[1]: Stopping Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:42:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[29062]: 2024-04-05T02:42:54.209+0000 7f9f0005e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T02:42:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[29062]: 2024-04-05T02:42:54.209+0000 7f9f0005e700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-05T02:42:54.697 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:54 smithi150 podman[29407]: 2024-04-05 02:42:54.394425068 +0000 UTC m=+0.271295807 container died 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-05T02:42:54.697 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:54 smithi150 podman[29407]: 2024-04-05 02:42:54.698456435 +0000 UTC m=+0.575327183 container cleanup 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-05T02:42:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:54 smithi150 bash[29407]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a 2024-04-05T02:42:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:54 smithi150 podman[29426]: 2024-04-05 02:42:54.899421047 +0000 UTC m=+0.508209305 container remove 5b462656771c5fc25b1877bb5cb755d0d06f8e46a383b1a44c5125a853a02355 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-05T02:42:55.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:55 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-05T02:42:55.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:55 smithi150 systemd[1]: Stopped Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:42:55.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:55 smithi150 systemd[1]: Starting Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:42:55.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:55 smithi150 podman[29519]: 2024-04-05 02:42:55.150970843 +0000 UTC m=+0.017578072 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-05T02:42:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:55 smithi150 podman[29519]: 2024-04-05 02:42:55.924477086 +0000 UTC m=+0.791084314 container create ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 podman[29519]: 2024-04-05 02:42:57.777063898 +0000 UTC m=+2.643671129 container init ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS) 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 podman[29519]: 2024-04-05 02:42:57.783635395 +0000 UTC m=+2.650242627 container start ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f) 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable), process ceph-mon, pid 7 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: pidfile_write: ignore empty --pid-file 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: load: jerasure load: lrc 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: RocksDB version: 6.15.5 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Compile date Apr 3 2024 2024-04-05T02:42:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: DB SUMMARY 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: DB Session ID: RFX7M9R712OB5NJR8VDC 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84223 ; 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.env: 0x563293ce2a80 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.fs: Posix File System 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.info_log: 0x563294ff2e80 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.statistics: (nil) 2024-04-05T02:42:58.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.use_fsync: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.db_log_dir: 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:42:58.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.write_buffer_manager: 0x563295d62900 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:42:58.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.unordered_write: 0 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.row_cache: None 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.wal_filter: None 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:42:58.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.preserve_deletes: 0 2024-04-05T02:42:58.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:42:58.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:42:58.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:42:58.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_open_files: -1 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Compression algorithms supported: 2024-04-05T02:42:58.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kZSTD supported: 0 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.merge_operator: 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_filter: None 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:42:58.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563295042f58) 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: index_type: 0 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: data_block_index_type: 0 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: index_shortening: 1 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: hash_index_allow_collision: 1 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: checksum: 1 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: no_block_cache: 0 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache: 0x563295075090 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:42:58.237 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache_options: 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: capacity : 536870912 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: num_shard_bits : 4 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: strict_capacity_limit : 0 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache_compressed: (nil) 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: persistent_cache: (nil) 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_size: 4096 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_size_deviation: 10 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_restart_interval: 16 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: index_block_restart_interval: 1 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: metadata_block_size: 4096 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: partition_filters: 0 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: use_delta_encoding: 1 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-05T02:42:58.238 INFO:journalctl@ceph.mon.a.smithi150.stdout: whole_key_filtering: 1 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: verify_compression: 0 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: format_version: 4 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: enable_index_compression: 1 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_align: 0 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression: NoCompression 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.num_levels: 7 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:42:58.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:42:58.240 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:42:58.241 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.arena_block_size: 4194304 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:42:58.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.ttl: 2592000 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:42:58.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712284977812550, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712284977813323, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80655, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78916, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73125, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712284977, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "2a890589-2981-4376-ab87-9c478357a40d", "db_session_id": "RFX7M9R712OB5NJR8VDC"}} 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712284977813683, "job": 1, "event": "recovery_finished"} 2024-04-05T02:42:58.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563295009180 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: DB pointer 0x5632950de000 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** DB Stats ** 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:42:58.245 INFO:journalctl@ceph.mon.a.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: L0 2/0 80.52 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: Sum 2/0 80.52 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:42:58.246 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative compaction: 0.00 GB write, 24.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval compaction: 0.00 GB write, 24.02 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: L0 2/0 80.52 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Sum 2/0 80.52 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:42:58.247 INFO:journalctl@ceph.mon.a.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 144.6 0.00 0.00 1 0.001 0 0 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative compaction: 0.00 GB write, 23.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: starting mon.a rank 0 at public addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] at bind addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:58.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???) e1 preinit fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).mds e1 new map 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).mds e1 print_map 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: e1 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: legacy client fscid: -1 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: No filesystems configured 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: monmap e1: 1 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:42:58.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: fsmap 2024-04-05T02:42:58.250 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: osdmap e1: 0 total, 0 up, 0 in 2024-04-05T02:42:58.250 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:57 smithi150 ceph-mon[29558]: mgrmap e1: no daemons active 2024-04-05T02:42:58.607 INFO:teuthology.orchestra.run.smithi150.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-05T02:42:58.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:58 smithi150 bash[29519]: ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 2024-04-05T02:42:58.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:42:58 smithi150 systemd[1]: Started Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:43:01.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:01 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3841015544' entity='client.admin' 2024-04-05T02:43:02.199 INFO:teuthology.orchestra.run.smithi150.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-05T02:43:02.199 INFO:teuthology.orchestra.run.smithi150.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-05T02:43:02.199 INFO:teuthology.orchestra.run.smithi150.stdout:Creating mgr... 2024-04-05T02:43:02.200 INFO:teuthology.orchestra.run.smithi150.stdout:Verifying port 9283 ... 2024-04-05T02:43:02.383 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y 2024-04-05T02:43:02.383 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Failed to reset failed state of unit ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service: Unit ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service not loaded. 2024-04-05T02:43:02.530 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4.target.wants/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service → /etc/systemd/system/ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@.service. 2024-04-05T02:43:02.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:02 smithi150 systemd[1]: Starting Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:43:03.083 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:02 smithi150 podman[29797]: 2024-04-05 02:43:02.668125852 +0000 UTC m=+0.017663566 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph:quincy 2024-04-05T02:43:03.083 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:02 smithi150 podman[29797]: 2024-04-05 02:43:02.772932532 +0000 UTC m=+0.122470287 container create be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403) 2024-04-05T02:43:03.083 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:02 smithi150 podman[29797]: 2024-04-05 02:43:02.98336133 +0000 UTC m=+0.332899040 container init be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=quincy-6fda41f, ceph=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-05T02:43:03.083 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:02 smithi150 podman[29797]: 2024-04-05 02:43:02.988684192 +0000 UTC m=+0.338221903 container start be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f) 2024-04-05T02:43:03.083 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:03 smithi150 bash[29797]: be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 2024-04-05T02:43:03.095 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:43:03.095 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout disabled 2024-04-05T02:43:03.103 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:43:03.103 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout inactive 2024-04-05T02:43:03.103 INFO:teuthology.orchestra.run.smithi150.stdout:firewalld.service is not enabled 2024-04-05T02:43:03.103 INFO:teuthology.orchestra.run.smithi150.stdout:Not possible to enable service . firewalld.service is not available 2024-04-05T02:43:03.112 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:43:03.112 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout disabled 2024-04-05T02:43:03.119 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:43:03.119 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout inactive 2024-04-05T02:43:03.119 INFO:teuthology.orchestra.run.smithi150.stdout:firewalld.service is not enabled 2024-04-05T02:43:03.119 INFO:teuthology.orchestra.run.smithi150.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-05T02:43:03.119 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for mgr to start... 2024-04-05T02:43:03.120 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for mgr... 2024-04-05T02:43:03.368 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:03 smithi150 systemd[1]: Started Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:43:03.691 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:03 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:03.512+0000 7f23b9dec1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:43:03.691 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:03 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:03.606+0000 7f23b9dec1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:43:03.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:03 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1029053217' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsid": "15e4009a-f2f6-11ee-b649-cb9ed24678a4", 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.253 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:04.254 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:43:04.255 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:43:04.255 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:43:04.255 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:43:04.255 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:43:04.255 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:43:04.258 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:43:04.259 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:42:50.693909+0000", 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:04.260 INFO:teuthology.orchestra.run.smithi150.stdout:mgr not available, waiting (1/15)... 2024-04-05T02:43:04.561 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.247+0000 7f23b9dec1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:43:04.561 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.335+0000 7f23b9dec1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:43:04.561 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.415+0000 7f23b9dec1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:43:04.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.561+0000 7f23b9dec1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:43:04.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.640+0000 7f23b9dec1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:43:04.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.805+0000 7f23b9dec1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:43:05.234 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:04.981+0000 7f23b9dec1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:43:05.234 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:05.062+0000 7f23b9dec1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:43:05.234 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:05.152+0000 7f23b9dec1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:43:05.235 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:05.234+0000 7f23b9dec1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:43:06.599 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:06 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:06.274+0000 7f23b9dec1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:43:06.599 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:06 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:06.406+0000 7f23b9dec1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:43:06.599 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:06 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:06.599+0000 7f23b9dec1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:43:06.881 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:06 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:06.833+0000 7f23b9dec1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:43:07.194 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:06 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2602781100' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:43:07.195 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:07.011+0000 7f23b9dec1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:43:07.195 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:07.194+0000 7f23b9dec1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsid": "15e4009a-f2f6-11ee-b649-cb9ed24678a4", 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:43:07.384 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:07.385 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:43:07.386 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:43:07.386 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:43:07.386 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:43:07.386 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:43:07.386 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:43:07.389 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:43:07.390 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:42:50.693909+0000", 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:07.391 INFO:teuthology.orchestra.run.smithi150.stdout:mgr not available, waiting (2/15)... 2024-04-05T02:43:07.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:07.452+0000 7f23b9dec1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:43:07.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:07.536+0000 7f23b9dec1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:43:08.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:07.921+0000 7f23b9dec1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:43:08.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:08 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:08.691+0000 7f23b9dec1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:43:09.301 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:08 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:08.984+0000 7f23b9dec1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:43:09.563 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:09.311+0000 7f23b9dec1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:43:09.563 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:09.511+0000 7f23b9dec1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:43:09.840 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: Activating manager daemon y 2024-04-05T02:43:09.840 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: mgrmap e2: y(active, starting, since 0.00291316s) 2024-04-05T02:43:09.840 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:43:09.840 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:43:09.840 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:43:09.841 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:43:09.841 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:43:09.841 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: Manager daemon y is now available 2024-04-05T02:43:09.841 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:43:09.841 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' 2024-04-05T02:43:09.841 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:43:09.842 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' 2024-04-05T02:43:09.842 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:09 smithi150 ceph-mon[29558]: from='mgr.14100 172.21.15.150:0/1390786670' entity='mgr.y' 2024-04-05T02:43:10.529 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:10.532 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:10.532 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsid": "15e4009a-f2f6-11ee-b649-cb9ed24678a4", 2024-04-05T02:43:10.532 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:43:10.533 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:43:10.534 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:43:10.539 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:43:10.540 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:10.541 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:42:50.693909+0000", 2024-04-05T02:43:10.541 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:10.541 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:10.541 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:43:10.541 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:10.541 INFO:teuthology.orchestra.run.smithi150.stdout:mgr not available, waiting (3/15)... 2024-04-05T02:43:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:10 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3353689847' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:43:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:10 smithi150 ceph-mon[29558]: mgrmap e3: y(active, since 1.00606s) 2024-04-05T02:43:12.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:12 smithi150 ceph-mon[29558]: mgrmap e4: y(active, since 2s) 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsid": "15e4009a-f2f6-11ee-b649-cb9ed24678a4", 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "health": { 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-05T02:43:13.816 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 0 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "a" 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "quorum_age": 15, 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-05T02:43:13.817 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-05T02:43:13.818 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-05T02:43:13.819 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "restful" 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ], 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "modified": "2024-04-05T02:42:50.693909+0000", 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout }, 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:13.820 INFO:teuthology.orchestra.run.smithi150.stdout:mgr is available 2024-04-05T02:43:13.934 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:13 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/161685592' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-05T02:43:14.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:14 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2151255200' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-05T02:43:15.043 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:15.043 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout [global] 2024-04-05T02:43:15.043 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout fsid = 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:43:15.043 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-05T02:43:15.043 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout [osd] 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-05T02:43:15.044 INFO:teuthology.orchestra.run.smithi150.stdout:Enabling cephadm module... 2024-04-05T02:43:16.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:15 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2412926952' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-05T02:43:16.087 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:15 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setuser ceph since I am not root 2024-04-05T02:43:16.087 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:15 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setgroup ceph since I am not root 2024-04-05T02:43:16.407 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:16.312+0000 7f6169bb21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:43:16.408 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:16.407+0000 7f6169bb21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:43:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:16 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2412926952' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-05T02:43:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:16 smithi150 ceph-mon[29558]: mgrmap e5: y(active, since 6s) 2024-04-05T02:43:17.317 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.060+0000 7f6169bb21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:43:17.318 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.149+0000 7f6169bb21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:43:17.318 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.233+0000 7f6169bb21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:43:17.563 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:17.563 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-05T02:43:17.563 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-05T02:43:17.563 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-05T02:43:17.563 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-05T02:43:17.563 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:17.564 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for the mgr to restart... 2024-04-05T02:43:17.564 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for mgr epoch 5... 2024-04-05T02:43:17.646 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.384+0000 7f6169bb21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:43:17.646 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.465+0000 7f6169bb21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:43:17.646 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.646+0000 7f6169bb21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:43:17.916 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3181838316' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-05T02:43:17.916 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.831+0000 7f6169bb21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:43:17.916 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:17.916+0000 7f6169bb21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:43:18.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:18 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:18.011+0000 7f6169bb21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:43:18.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:18 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:18.099+0000 7f6169bb21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:43:19.452 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:19 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:19.132+0000 7f6169bb21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:43:19.453 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:19 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:19.261+0000 7f6169bb21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:43:19.453 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:19 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:19.452+0000 7f6169bb21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:43:19.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:19 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:19.686+0000 7f6169bb21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:43:20.162 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:19 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:19.857+0000 7f6169bb21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:43:20.162 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:20.035+0000 7f6169bb21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:43:20.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:20.296+0000 7f6169bb21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:43:20.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:20.380+0000 7f6169bb21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:43:21.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:20.768+0000 7f6169bb21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:43:21.825 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:21.535+0000 7f6169bb21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:43:21.825 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:21.825+0000 7f6169bb21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:43:22.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:22.141+0000 7f6169bb21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:43:22.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:22.331+0000 7f6169bb21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:43:22.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: Active manager daemon y restarted 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: Activating manager daemon y 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: osdmap e2: 0 total, 0 up, 0 in 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: mgrmap e6: y(active, starting, since 0.00324424s) 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: Manager daemon y is now available 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:22.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:22.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:43:22.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:22 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:43:23.716 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:23 smithi150 ceph-mon[29558]: Found migration_current of "None". Setting to last migration. 2024-04-05T02:43:23.716 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:23 smithi150 ceph-mon[29558]: mgrmap e7: y(active, since 1.00479s) 2024-04-05T02:43:24.012 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:24.012 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-05T02:43:24.012 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-05T02:43:24.012 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:24.012 INFO:teuthology.orchestra.run.smithi150.stdout:mgr epoch 5 is available 2024-04-05T02:43:24.012 INFO:teuthology.orchestra.run.smithi150.stdout:Setting orchestrator backend to cephadm... 2024-04-05T02:43:24.655 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-05T02:43:24.656 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: [05/Apr/2024:02:43:23] ENGINE Bus STARTING 2024-04-05T02:43:24.656 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-05T02:43:24.656 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: [05/Apr/2024:02:43:23] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:43:24.656 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: [05/Apr/2024:02:43:23] ENGINE Bus STARTED 2024-04-05T02:43:24.656 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:24.656 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:24 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:25.675 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:25 smithi150 ceph-mon[29558]: mgrmap e8: y(active, since 2s) 2024-04-05T02:43:25.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:25 smithi150 ceph-mon[29558]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:25.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:25 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:25.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:25 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:26.415 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-05T02:43:26.415 INFO:teuthology.orchestra.run.smithi150.stdout:Generating ssh key... 2024-04-05T02:43:27.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:26 smithi150 ceph-mon[29558]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:27.601 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: Generating public/private rsa key pair. 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: Your identification has been saved in /tmp/tmp7_0cfbm0/key. 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: Your public key has been saved in /tmp/tmp7_0cfbm0/key.pub. 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: The key fingerprint is: 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: SHA256:RiyMMb7f0dtlWRVOtA0ue606lKvnK3HMjrTnu4bGVbU ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: The key's randomart image is: 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: +---[RSA 3072]----+ 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | o o+o| 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | . = . .o.=| 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | o o o . .++| 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | . o . o.E | 2024-04-05T02:43:27.602 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | . S .o.o= .| 2024-04-05T02:43:27.603 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | . o .ooB+ . | 2024-04-05T02:43:27.603 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | . .o.X... | 2024-04-05T02:43:27.603 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | B B. | 2024-04-05T02:43:27.603 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: | ..OB= | 2024-04-05T02:43:27.603 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: +----[SHA256]-----+ 2024-04-05T02:43:28.470 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:28 smithi150 ceph-mon[29558]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:28.470 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:28 smithi150 ceph-mon[29558]: Generating ssh key... 2024-04-05T02:43:28.470 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:28 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:28.470 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:28 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:29.002 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDC1ckvfKfZ6hRg0l66v/il/jYQ5skW1OCmQg2vPNYRHylyUwXS43gvo3KDx9V7dGJBhaysYvHoCln5cmxE81UOQIXO7A8g8Q1CErE9km+QJtIT+jhc62BP08bykUmigdgEVXqVQG1eFwaeXbXdnB4mPEo/vuKdiopkXZhp8dnTBwPzhjwD3sc606op0isZyFkbMIJW2qPeWezFMpP2MbYZ6UrfWuxZDx3XYfE1sREC28JZYZpRDiRUppIXgvBUxYw8suG7YZXdmgFH/cy70mWGRr/Wpr0PYl4fhdI1/G9+6YD5+JyrUqlr5Bgy9PByuSXDKf9wXxVi2rPqZOiTdW1X/ziQCqlMfIIaen+eUivtjS9niTuaMOugJ7gPcx25p8zFn07nLEFE7e7aN6v7iXQaNUTnMuQF+wBGeWjjxfrQzh+GMuVOPP1c/uwsR/Ck9tRgR4APcDWDOTReDA+FhHSCKNpjRw4C60GMeRc/Q7TJD0zrACcUvjLHZVuMGhFm2WM= ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:43:29.002 INFO:teuthology.orchestra.run.smithi150.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-05T02:43:29.002 INFO:teuthology.orchestra.run.smithi150.stdout:Adding key to root@localhost authorized_keys... 2024-04-05T02:43:29.003 INFO:teuthology.orchestra.run.smithi150.stdout:Adding host smithi150... 2024-04-05T02:43:29.660 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:29 smithi150 ceph-mon[29558]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:30.635 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:30 smithi150 ceph-mon[29558]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi150", "addr": "172.21.15.150", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:31.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:31 smithi150 ceph-mon[29558]: Deploying cephadm binary to smithi150 2024-04-05T02:43:32.513 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout Added host 'smithi150' with addr '172.21.15.150' 2024-04-05T02:43:32.513 INFO:teuthology.orchestra.run.smithi150.stdout:Deploying unmanaged mon service... 2024-04-05T02:43:32.780 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:32 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:32.780 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:32 smithi150 ceph-mon[29558]: Added host smithi150 2024-04-05T02:43:32.780 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:32 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:34.517 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-05T02:43:34.517 INFO:teuthology.orchestra.run.smithi150.stdout:Deploying unmanaged mgr service... 2024-04-05T02:43:34.624 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:34 smithi150 ceph-mon[29558]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:34.624 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:34 smithi150 ceph-mon[29558]: Saving service mon spec with placement count:5 2024-04-05T02:43:34.624 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:34 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:34.624 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:34 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:35.956 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:35 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:35.956 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:35 smithi150 ceph-mon[29558]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:35.956 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:35 smithi150 ceph-mon[29558]: Saving service mgr spec with placement count:2 2024-04-05T02:43:35.956 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:35 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:35.958 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-05T02:43:37.876 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:37 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:37.877 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:37 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2250164378' entity='client.admin' 2024-04-05T02:43:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:38 smithi150 ceph-mon[29558]: from='mgr.14120 172.21.15.150:0/127861261' entity='mgr.y' 2024-04-05T02:43:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:38 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1101082332' entity='client.admin' 2024-04-05T02:43:39.253 INFO:teuthology.orchestra.run.smithi150.stdout:Enabling the dashboard module... 2024-04-05T02:43:40.567 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:40 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3723128610' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-05T02:43:40.567 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setuser ceph since I am not root 2024-04-05T02:43:40.567 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setgroup ceph since I am not root 2024-04-05T02:43:40.842 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:40.812+0000 7fe7821461c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:43:41.187 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:40.907+0000 7fe7821461c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:43:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3723128610' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-05T02:43:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-mon[29558]: mgrmap e9: y(active, since 17s) 2024-04-05T02:43:41.884 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:41.556+0000 7fe7821461c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:43:41.884 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:41.647+0000 7fe7821461c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:43:41.884 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:41.730+0000 7fe7821461c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:43:41.884 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:41.884+0000 7fe7821461c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:43:42.137 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:41.966+0000 7fe7821461c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:43:42.137 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:42 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:42.137+0000 7fe7821461c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:43:42.168 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:42.168 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-05T02:43:42.168 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-05T02:43:42.168 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-05T02:43:42.168 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-05T02:43:42.168 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:42.169 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for the mgr to restart... 2024-04-05T02:43:42.169 INFO:teuthology.orchestra.run.smithi150.stdout:Waiting for mgr epoch 9... 2024-04-05T02:43:42.398 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:42 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4144018129' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-05T02:43:42.398 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:42 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:42.326+0000 7fe7821461c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:43:42.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:42 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:42.415+0000 7fe7821461c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:43:42.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:42 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:42.511+0000 7fe7821461c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:43:42.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:42 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:42.600+0000 7fe7821461c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:43:43.961 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:43.641+0000 7fe7821461c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:43:43.961 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:43.769+0000 7fe7821461c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:43:43.961 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:43.960+0000 7fe7821461c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:43:44.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:44.188+0000 7fe7821461c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:43:44.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:44.361+0000 7fe7821461c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:43:44.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:44.540+0000 7fe7821461c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:43:45.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:44.799+0000 7fe7821461c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:43:45.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:44.884+0000 7fe7821461c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:43:45.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:45 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:45.290+0000 7fe7821461c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:43:46.349 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:46.059+0000 7fe7821461c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:43:46.349 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:46.349+0000 7fe7821461c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:46.663+0000 7fe7821461c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:43:46.855+0000 7fe7821461c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: Active manager daemon y restarted 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: Activating manager daemon y 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: osdmap e3: 0 total, 0 up, 0 in 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: mgrmap e10: y(active, starting, since 0.00319181s) 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:43:46.963 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:43:46.964 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:43:46.964 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: Manager daemon y is now available 2024-04-05T02:43:46.964 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:46.964 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:43:46.964 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:43:46.964 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:46 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:43:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:47 smithi150 ceph-mon[29558]: [05/Apr/2024:02:43:47] ENGINE Bus STARTING 2024-04-05T02:43:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:43:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:47 smithi150 ceph-mon[29558]: mgrmap e11: y(active, since 1.00552s) 2024-04-05T02:43:48.569 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout { 2024-04-05T02:43:48.569 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-05T02:43:48.569 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-05T02:43:48.569 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout } 2024-04-05T02:43:48.569 INFO:teuthology.orchestra.run.smithi150.stdout:mgr epoch 9 is available 2024-04-05T02:43:48.569 INFO:teuthology.orchestra.run.smithi150.stdout:Generating a dashboard self-signed certificate... 2024-04-05T02:43:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:49 smithi150 ceph-mon[29558]: [05/Apr/2024:02:43:47] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:43:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:49 smithi150 ceph-mon[29558]: [05/Apr/2024:02:43:47] ENGINE Bus STARTED 2024-04-05T02:43:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:49 smithi150 ceph-mon[29558]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-05T02:43:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:49 smithi150 ceph-mon[29558]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-05T02:43:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:43:49.892 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-05T02:43:49.892 INFO:teuthology.orchestra.run.smithi150.stdout:Creating initial admin user... 2024-04-05T02:43:50.265 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:50 smithi150 ceph-mon[29558]: mgrmap e12: y(active, since 2s) 2024-04-05T02:43:50.265 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:50 smithi150 ceph-mon[29558]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:50.265 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:43:50.265 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:43:50.265 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:43:52.152 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:51 smithi150 ceph-mon[29558]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:43:52.152 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:51 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:43:52.155 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$HwYA/zJJo1UHSPu5UkaSYezmKx.cVJjYWtaoZ85.LVcz2dliNztU2", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712285030, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-05T02:43:52.155 INFO:teuthology.orchestra.run.smithi150.stdout:Fetching dashboard port number... 2024-04-05T02:43:53.116 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:52 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1291804143' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-05T02:43:53.369 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stdout 8443 2024-04-05T02:43:53.379 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-05T02:43:53.379 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout disabled 2024-04-05T02:43:53.387 INFO:teuthology.orchestra.run.smithi150.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-05T02:43:53.387 INFO:teuthology.orchestra.run.smithi150.stdout:systemctl: stdout inactive 2024-04-05T02:43:53.387 INFO:teuthology.orchestra.run.smithi150.stdout:firewalld.service is not enabled 2024-04-05T02:43:53.387 INFO:teuthology.orchestra.run.smithi150.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-05T02:43:53.389 INFO:teuthology.orchestra.run.smithi150.stdout:Ceph Dashboard is now available at: 2024-04-05T02:43:53.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:53.389 INFO:teuthology.orchestra.run.smithi150.stdout: URL: https://smithi150.front.sepia.ceph.com:8443/ 2024-04-05T02:43:53.389 INFO:teuthology.orchestra.run.smithi150.stdout: User: admin 2024-04-05T02:43:53.389 INFO:teuthology.orchestra.run.smithi150.stdout: Password: koeftzxoux 2024-04-05T02:43:53.389 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:53.390 INFO:teuthology.orchestra.run.smithi150.stdout:Saving cluster configuration to /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config directory 2024-04-05T02:43:53.390 INFO:teuthology.orchestra.run.smithi150.stdout:Enabling autotune for osd_memory_target 2024-04-05T02:43:54.934 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:54 smithi150 ceph-mon[29558]: mgrmap e13: y(active, since 6s) 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout:Or, if you are only running a single cluster on this host: 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.874 INFO:teuthology.orchestra.run.smithi150.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: ceph telemetry on 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout:For more information see: 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:43:55.875 INFO:teuthology.orchestra.run.smithi150.stdout:Bootstrap complete. 2024-04-05T02:43:55.913 INFO:tasks.cephadm:Fetching config... 2024-04-05T02:43:55.913 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:43:55.913 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-05T02:43:55.928 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-05T02:43:55.929 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:43:55.929 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-05T02:43:55.986 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-05T02:43:55.987 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:43:55.987 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/keyring of=/dev/stdout 2024-04-05T02:43:56.054 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-05T02:43:56.054 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:43:56.054 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-05T02:43:56.109 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-05T02:43:56.109 DEBUG:teuthology.orchestra.run.smithi150:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDC1ckvfKfZ6hRg0l66v/il/jYQ5skW1OCmQg2vPNYRHylyUwXS43gvo3KDx9V7dGJBhaysYvHoCln5cmxE81UOQIXO7A8g8Q1CErE9km+QJtIT+jhc62BP08bykUmigdgEVXqVQG1eFwaeXbXdnB4mPEo/vuKdiopkXZhp8dnTBwPzhjwD3sc606op0isZyFkbMIJW2qPeWezFMpP2MbYZ6UrfWuxZDx3XYfE1sREC28JZYZpRDiRUppIXgvBUxYw8suG7YZXdmgFH/cy70mWGRr/Wpr0PYl4fhdI1/G9+6YD5+JyrUqlr5Bgy9PByuSXDKf9wXxVi2rPqZOiTdW1X/ziQCqlMfIIaen+eUivtjS9niTuaMOugJ7gPcx25p8zFn07nLEFE7e7aN6v7iXQaNUTnMuQF+wBGeWjjxfrQzh+GMuVOPP1c/uwsR/Ck9tRgR4APcDWDOTReDA+FhHSCKNpjRw4C60GMeRc/Q7TJD0zrACcUvjLHZVuMGhFm2WM= ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-05T02:43:56.206 INFO:teuthology.orchestra.run.smithi150.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDC1ckvfKfZ6hRg0l66v/il/jYQ5skW1OCmQg2vPNYRHylyUwXS43gvo3KDx9V7dGJBhaysYvHoCln5cmxE81UOQIXO7A8g8Q1CErE9km+QJtIT+jhc62BP08bykUmigdgEVXqVQG1eFwaeXbXdnB4mPEo/vuKdiopkXZhp8dnTBwPzhjwD3sc606op0isZyFkbMIJW2qPeWezFMpP2MbYZ6UrfWuxZDx3XYfE1sREC28JZYZpRDiRUppIXgvBUxYw8suG7YZXdmgFH/cy70mWGRr/Wpr0PYl4fhdI1/G9+6YD5+JyrUqlr5Bgy9PByuSXDKf9wXxVi2rPqZOiTdW1X/ziQCqlMfIIaen+eUivtjS9niTuaMOugJ7gPcx25p8zFn07nLEFE7e7aN6v7iXQaNUTnMuQF+wBGeWjjxfrQzh+GMuVOPP1c/uwsR/Ck9tRgR4APcDWDOTReDA+FhHSCKNpjRw4C60GMeRc/Q7TJD0zrACcUvjLHZVuMGhFm2WM= ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:43:56.221 DEBUG:teuthology.orchestra.run.smithi196:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDC1ckvfKfZ6hRg0l66v/il/jYQ5skW1OCmQg2vPNYRHylyUwXS43gvo3KDx9V7dGJBhaysYvHoCln5cmxE81UOQIXO7A8g8Q1CErE9km+QJtIT+jhc62BP08bykUmigdgEVXqVQG1eFwaeXbXdnB4mPEo/vuKdiopkXZhp8dnTBwPzhjwD3sc606op0isZyFkbMIJW2qPeWezFMpP2MbYZ6UrfWuxZDx3XYfE1sREC28JZYZpRDiRUppIXgvBUxYw8suG7YZXdmgFH/cy70mWGRr/Wpr0PYl4fhdI1/G9+6YD5+JyrUqlr5Bgy9PByuSXDKf9wXxVi2rPqZOiTdW1X/ziQCqlMfIIaen+eUivtjS9niTuaMOugJ7gPcx25p8zFn07nLEFE7e7aN6v7iXQaNUTnMuQF+wBGeWjjxfrQzh+GMuVOPP1c/uwsR/Ck9tRgR4APcDWDOTReDA+FhHSCKNpjRw4C60GMeRc/Q7TJD0zrACcUvjLHZVuMGhFm2WM= ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-05T02:43:56.266 INFO:teuthology.orchestra.run.smithi196.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDC1ckvfKfZ6hRg0l66v/il/jYQ5skW1OCmQg2vPNYRHylyUwXS43gvo3KDx9V7dGJBhaysYvHoCln5cmxE81UOQIXO7A8g8Q1CErE9km+QJtIT+jhc62BP08bykUmigdgEVXqVQG1eFwaeXbXdnB4mPEo/vuKdiopkXZhp8dnTBwPzhjwD3sc606op0isZyFkbMIJW2qPeWezFMpP2MbYZ6UrfWuxZDx3XYfE1sREC28JZYZpRDiRUppIXgvBUxYw8suG7YZXdmgFH/cy70mWGRr/Wpr0PYl4fhdI1/G9+6YD5+JyrUqlr5Bgy9PByuSXDKf9wXxVi2rPqZOiTdW1X/ziQCqlMfIIaen+eUivtjS9niTuaMOugJ7gPcx25p8zFn07nLEFE7e7aN6v7iXQaNUTnMuQF+wBGeWjjxfrQzh+GMuVOPP1c/uwsR/Ck9tRgR4APcDWDOTReDA+FhHSCKNpjRw4C60GMeRc/Q7TJD0zrACcUvjLHZVuMGhFm2WM= ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:43:56.281 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-05T02:43:56.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:43:56 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1189710654' entity='client.admin' 2024-04-05T02:43:56.538 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:43:59.753 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-05T02:43:59.754 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-05T02:44:00.008 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:00.357 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:00 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3158458823' entity='client.admin' 2024-04-05T02:44:01.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi150", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:44:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:02.883 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi196 2024-04-05T02:44:02.884 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:44:02.884 DEBUG:teuthology.orchestra.run.smithi196:> dd of=/etc/ceph/ceph.conf 2024-04-05T02:44:02.902 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:44:02.902 DEBUG:teuthology.orchestra.run.smithi196:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:02.958 INFO:tasks.cephadm:Adding host smithi196 to orchestrator... 2024-04-05T02:44:02.959 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch host add smithi196 2024-04-05T02:44:03.214 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:44:03.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:03.426 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:03.426 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:03 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:04.426 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:04 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:04.427 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:04 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:04.427 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:04 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:44:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:05 smithi150 ceph-mon[29558]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi196", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:06.644 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:06 smithi150 ceph-mon[29558]: Deploying cephadm binary to smithi196 2024-04-05T02:44:06.952 INFO:teuthology.orchestra.run.smithi150.stdout:Added host 'smithi196' with addr '172.21.15.196' 2024-04-05T02:44:07.511 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-05T02:44:07.782 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:08.154 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:07 smithi150 ceph-mon[29558]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:08.154 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:08.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:07 smithi150 ceph-mon[29558]: Added host smithi196 2024-04-05T02:44:08.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:08.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:08.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:09.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:09.562 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:44:09.563 INFO:teuthology.orchestra.run.smithi150.stdout:[{"addr": "172.21.15.150", "hostname": "smithi150", "labels": [], "status": ""}, {"addr": "172.21.15.196", "hostname": "smithi196", "labels": [], "status": ""}] 2024-04-05T02:44:10.147 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-05T02:44:10.148 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-05T02:44:10.395 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:10.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:10 smithi150 ceph-mon[29558]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi196", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:11.445 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:12 smithi150 ceph-mon[29558]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:12 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:44:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:12 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:12 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:12.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:12 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:12.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:12 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2005436314' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-05T02:44:13.149 INFO:teuthology.orchestra.run.smithi150.stderr:adjusted tunables profile to default 2024-04-05T02:44:13.451 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:13 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:13.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:13 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:13.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:13 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:44:13.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:13 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2005436314' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-05T02:44:13.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:13 smithi150 ceph-mon[29558]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:13.755 INFO:tasks.cephadm:Adding mon.a on smithi150 2024-04-05T02:44:13.755 INFO:tasks.cephadm:Adding mon.c on smithi150 2024-04-05T02:44:13.755 INFO:tasks.cephadm:Adding mon.b on smithi196 2024-04-05T02:44:13.755 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch apply mon '3;smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b' 2024-04-05T02:44:14.003 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:14.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:14 smithi150 ceph-mon[29558]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:15.686 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled mon update... 2024-04-05T02:44:16.348 DEBUG:teuthology.orchestra.run.smithi150:mon.c> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.c.service 2024-04-05T02:44:16.351 DEBUG:teuthology.orchestra.run.smithi196:mon.b> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.b.service 2024-04-05T02:44:16.353 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-05T02:44:16.353 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-05T02:44:16.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:16.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:16.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:16.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:16 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:17.222 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:17 smithi150 ceph-mon[29558]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:17 smithi150 ceph-mon[29558]: Saving service mon spec with placement smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b;count:3 2024-04-05T02:44:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:17 smithi150 ceph-mon[29558]: Deploying daemon mon.b on smithi196 2024-04-05T02:44:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:18 smithi150 ceph-mon[29558]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:20.411 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:44:20.412 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":1,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","modified":"2024-04-05T02:42:41.704408Z","created":"2024-04-05T02:42:41.704408Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3300","nonce":0},{"type":"v1","addr":"172.21.15.150:6789","nonce":0}]},"addr":"172.21.15.150:6789/0","public_addr":"172.21.15.150:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-05T02:44:20.414 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 1 2024-04-05T02:44:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:20 smithi196 systemd[1]: Starting Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:44:20.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:20 smithi150 ceph-mon[29558]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:21.276 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 podman[33131]: 2024-04-05 02:44:21.056240296 +0000 UTC m=+0.017704678 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T02:44:21.276 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 podman[33131]: 2024-04-05 02:44:21.156530334 +0000 UTC m=+0.117994714 container create 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD) 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 podman[33131]: 2024-04-05 02:44:21.350674233 +0000 UTC m=+0.312138612 container init 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2) 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 podman[33131]: 2024-04-05 02:44:21.355624567 +0000 UTC m=+0.317088949 container start 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable), process ceph-mon, pid 7 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: pidfile_write: ignore empty --pid-file 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: load: jerasure load: lrc 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: RocksDB version: 6.15.5 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Compile date Apr 3 2024 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: DB SUMMARY 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: DB Session ID: CJV91FROX9ENS98EMJK7 2024-04-05T02:44:21.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.env: 0x560c181e2a80 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.fs: Posix File System 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.info_log: 0x560c19a54e80 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.statistics: (nil) 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.use_fsync: 0 2024-04-05T02:44:21.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.db_log_dir: 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:44:21.640 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.write_buffer_manager: 0x560c1a7c4900 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:44:21.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.unordered_write: 0 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.row_cache: None 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.wal_filter: None 2024-04-05T02:44:21.642 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.preserve_deletes: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:44:21.646 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:44:21.647 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_open_files: -1 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Compression algorithms supported: 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:44:21.648 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kZSTD supported: 0 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.merge_operator: 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_filter: None 2024-04-05T02:44:21.649 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560c19aa4f58) 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:44:21.650 INFO:journalctl@ceph.mon.b.smithi196.stdout: index_type: 0 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: data_block_index_type: 0 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: index_shortening: 1 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: hash_index_allow_collision: 1 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: checksum: 1 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: no_block_cache: 0 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache: 0x560c19ad7090 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:44:21.651 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache_options: 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: capacity : 536870912 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: num_shard_bits : 4 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: strict_capacity_limit : 0 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache_compressed: (nil) 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: persistent_cache: (nil) 2024-04-05T02:44:21.652 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_size: 4096 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_size_deviation: 10 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_restart_interval: 16 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: index_block_restart_interval: 1 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: metadata_block_size: 4096 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: partition_filters: 0 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: use_delta_encoding: 1 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: whole_key_filtering: 1 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: verify_compression: 0 2024-04-05T02:44:21.653 INFO:journalctl@ceph.mon.b.smithi196.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:44:21.654 INFO:journalctl@ceph.mon.b.smithi196.stdout: format_version: 4 2024-04-05T02:44:21.654 INFO:journalctl@ceph.mon.b.smithi196.stdout: enable_index_compression: 1 2024-04-05T02:44:21.654 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_align: 0 2024-04-05T02:44:21.654 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:44:21.654 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:44:21.654 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression: NoCompression 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.num_levels: 7 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:44:21.655 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:44:21.656 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:44:21.657 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:44:21.658 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.arena_block_size: 4194304 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:44:21.659 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:44:21.660 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:44:21.661 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.ttl: 2592000 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-05T02:44:21.662 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285061382881, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285061383216, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712285061, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "64bb1cb2-7c84-45f0-9f38-0bc83ad7c3a4", "db_session_id": "CJV91FROX9ENS98EMJK7"}} 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285061383549, "job": 1, "event": "recovery_finished"} 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x560c19a6b180 2024-04-05T02:44:21.663 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: DB pointer 0x560c19b40000 2024-04-05T02:44:21.664 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-05T02:44:21.664 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-05T02:44:21.664 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-05T02:44:21.664 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** DB Stats ** 2024-04-05T02:44:21.664 INFO:journalctl@ceph.mon.b.smithi196.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.665 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:21.666 INFO:journalctl@ceph.mon.b.smithi196.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:21.666 INFO:journalctl@ceph.mon.b.smithi196.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:21.666 INFO:journalctl@ceph.mon.b.smithi196.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.667 INFO:journalctl@ceph.mon.b.smithi196.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.667 INFO:journalctl@ceph.mon.b.smithi196.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.667 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.667 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:21.667 INFO:journalctl@ceph.mon.b.smithi196.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:44:21.668 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.669 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:44:21.670 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: using public_addr v2:172.21.15.196:0/0 -> [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: starting mon.b rank -1 at public addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] at bind addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(???) e0 preinit fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).mds e1 new map 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: e1 2024-04-05T02:44:21.671 INFO:journalctl@ceph.mon.b.smithi196.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout: legacy client fscid: -1 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout: No filesystems configured 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-05T02:44:21.672 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi196", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:21.673 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2005436314' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:44:21.674 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2005436314' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:21.675 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: Saving service mon spec with placement smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b;count:3 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: Deploying daemon mon.b on smithi196 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/819066712' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 ceph-mon[33169]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 bash[33131]: 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c 2024-04-05T02:44:21.676 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:21 smithi196 systemd[1]: Started Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:44:21.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:21 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/819066712' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:21.963 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-05T02:44:21.963 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-05T02:44:22.214 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:44:26.840 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:44:26.840 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: mon.a calling monitor election 2024-04-05T02:44:26.840 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.840 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: mon.b calling monitor election 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: monmap e2: 2 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],b=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: fsmap 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:26.841 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: mgrmap e13: y(active, since 39s) 2024-04-05T02:44:26.842 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: overall HEALTH_OK 2024-04-05T02:44:26.842 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:26.842 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:26.912 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:44:26.912 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: mon.a calling monitor election 2024-04-05T02:44:26.912 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.912 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.912 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:26.912 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: mon.b calling monitor election 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: monmap e2: 2 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],b=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: fsmap 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:26.913 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: mgrmap e13: y(active, since 39s) 2024-04-05T02:44:26.914 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: overall HEALTH_OK 2024-04-05T02:44:26.914 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:26.914 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:27.049 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:44:27.049 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":2,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","modified":"2024-04-05T02:44:21.404393Z","created":"2024-04-05T02:42:41.704408Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3300","nonce":0},{"type":"v1","addr":"172.21.15.150:6789","nonce":0}]},"addr":"172.21.15.150:6789/0","public_addr":"172.21.15.150:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:3300","nonce":0},{"type":"v1","addr":"172.21.15.196:6789","nonce":0}]},"addr":"172.21.15.196:6789/0","public_addr":"172.21.15.196:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-05T02:44:27.052 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 2 2024-04-05T02:44:27.594 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:27.595 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:27.595 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:27.595 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: Deploying daemon mon.c on smithi150 2024-04-05T02:44:27.595 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:27.595 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/3930750444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:27.595 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:27 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:27.940 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:27.940 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:27.941 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:27.941 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: Deploying daemon mon.c on smithi150 2024-04-05T02:44:27.941 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:27.941 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/3930750444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:27.941 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:27 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:28.621 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-05T02:44:28.621 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph mon dump -f json 2024-04-05T02:44:28.871 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:44:29.464 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 systemd[1]: Starting Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:44:29.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 podman[41780]: 2024-04-05 02:44:29.368305522 +0000 UTC m=+0.017879414 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 podman[41780]: 2024-04-05 02:44:29.490756097 +0000 UTC m=+0.140330007 container create 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-6fda41f, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 podman[41780]: 2024-04-05 02:44:29.668228703 +0000 UTC m=+0.317802574 container init 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f) 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 podman[41780]: 2024-04-05 02:44:29.673290393 +0000 UTC m=+0.322864264 container start 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True) 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable), process ceph-mon, pid 7 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pidfile_write: ignore empty --pid-file 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: load: jerasure load: lrc 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: RocksDB version: 6.15.5 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Compile date Apr 3 2024 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: DB SUMMARY 2024-04-05T02:44:29.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: DB Session ID: 41PGNY0DJ2GHWY725ZBB 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.env: 0x55d861e65a80 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.fs: Posix File System 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.info_log: 0x55d862f04e80 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.statistics: (nil) 2024-04-05T02:44:29.720 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.use_fsync: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.db_log_dir: 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:44:29.721 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.write_buffer_manager: 0x55d863c74930 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:44:29.722 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.unordered_write: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.row_cache: None 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.wal_filter: None 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.preserve_deletes: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:44:29.723 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:44:29.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_open_files: -1 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Compression algorithms supported: 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:44:29.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kZSTD supported: 0 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.merge_operator: 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_filter: None 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:44:29.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d862f54f58) 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: index_type: 0 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: data_block_index_type: 0 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: index_shortening: 1 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: hash_index_allow_collision: 1 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: checksum: 1 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: no_block_cache: 0 2024-04-05T02:44:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache: 0x55d862f87090 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache_options: 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: capacity : 536870912 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: num_shard_bits : 4 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: strict_capacity_limit : 0 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache_compressed: (nil) 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: persistent_cache: (nil) 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_size: 4096 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_size_deviation: 10 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_restart_interval: 16 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: index_block_restart_interval: 1 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: metadata_block_size: 4096 2024-04-05T02:44:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout: partition_filters: 0 2024-04-05T02:44:29.729 INFO:journalctl@ceph.mon.c.smithi150.stdout: use_delta_encoding: 1 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: whole_key_filtering: 1 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: verify_compression: 0 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: format_version: 4 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: enable_index_compression: 1 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_align: 0 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression: NoCompression 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:44:29.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.num_levels: 7 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:44:29.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:44:29.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.arena_block_size: 4194304 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:44:29.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:44:29.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.ttl: 2592000 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-05T02:44:29.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285069699880, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285069700191, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712285069, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "bbac959f-e8fb-41cd-9fa9-6cae06e2a51d", "db_session_id": "41PGNY0DJ2GHWY725ZBB"}} 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285069700493, "job": 1, "event": "recovery_finished"} 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d862f1b180 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: DB pointer 0x55d862ff0000 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** DB Stats ** 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:44:29.736 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:29.737 INFO:journalctl@ceph.mon.c.smithi150.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:44:29.738 INFO:journalctl@ceph.mon.c.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.739 INFO:journalctl@ceph.mon.c.smithi150.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.4 0.00 0.00 1 0.000 0 0 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:44:29.740 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: using public_addrv [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: starting mon.c rank -1 at public addrs [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] at bind addrs [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(???) e0 preinit fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).mds e1 new map 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).mds e1 print_map 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: e1 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:44:29.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: legacy client fscid: -1 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: No filesystems configured 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-05T02:44:29.742 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi196", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2005436314' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:29.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:29.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2005436314' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:29.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Saving service mon spec with placement smithi150:172.21.15.150=a;smithi150:[v2:172.21.15.150:3301,v1:172.21.15.150:6790]=c;smithi196:172.21.15.196=b;count:3 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Deploying daemon mon.b on smithi196 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/819066712' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.a calling monitor election 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.b calling monitor election 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: monmap e2: 2 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],b=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: fsmap 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mgrmap e13: y(active, since 39s) 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: overall HEALTH_OK 2024-04-05T02:44:29.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: Deploying daemon mon.c on smithi150 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/3930750444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:29.981 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 ceph-mon[41818]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-05T02:44:29.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 bash[41780]: 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 2024-04-05T02:44:29.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:29 smithi150 systemd[1]: Started Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:44:35.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: mon.a calling monitor election 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: mon.b calling monitor election 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: mon.c calling monitor election 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:44:35.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: monmap e3: 3 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],b=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0],c=[v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: fsmap 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: mgrmap e13: y(active, since 47s) 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: overall HEALTH_OK 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:44:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: mon.a calling monitor election 2024-04-05T02:44:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: mon.b calling monitor election 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: mon.c calling monitor election 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: monmap e3: 3 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],b=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0],c=[v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: fsmap 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: mgrmap e13: y(active, since 47s) 2024-04-05T02:44:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: overall HEALTH_OK 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: mon.a calling monitor election 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: mon.b calling monitor election 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: mon.c calling monitor election 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: monmap e3: 3 mons at {a=[v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0],b=[v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0],c=[v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: fsmap 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: osdmap e4: 0 total, 0 up, 0 in 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: mgrmap e13: y(active, since 47s) 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: overall HEALTH_OK 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:35.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:36.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:35 smithi196 ceph-mon[33169]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:35 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:35 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:35 smithi150 ceph-mon[29558]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:35 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:35 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:36.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:35 smithi150 ceph-mon[41818]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:36.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:35 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:35 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:44:36.556 INFO:teuthology.orchestra.run.smithi196.stdout: 2024-04-05T02:44:36.556 INFO:teuthology.orchestra.run.smithi196.stdout:{"epoch":3,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","modified":"2024-04-05T02:44:29.726178Z","created":"2024-04-05T02:42:41.704408Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3300","nonce":0},{"type":"v1","addr":"172.21.15.150:6789","nonce":0}]},"addr":"172.21.15.150:6789/0","public_addr":"172.21.15.150:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:3300","nonce":0},{"type":"v1","addr":"172.21.15.196:6789","nonce":0}]},"addr":"172.21.15.196:6789/0","public_addr":"172.21.15.196:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:3301","nonce":0},{"type":"v1","addr":"172.21.15.150:6790","nonce":0}]},"addr":"172.21.15.150:6790/0","public_addr":"172.21.15.150:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-05T02:44:36.558 INFO:teuthology.orchestra.run.smithi196.stderr:dumped monmap epoch 3 2024-04-05T02:44:37.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:37.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/2667859604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:37.085 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.086 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/2667859604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:37.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:37.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:37.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:37.100 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.101 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:37.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/2667859604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-05T02:44:37.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:37.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:37.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:37.124 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-05T02:44:37.124 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-05T02:44:38.011 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:38.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:37 smithi196 ceph-mon[33169]: Reconfiguring mon.a (unknown last config time)... 2024-04-05T02:44:38.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:37 smithi196 ceph-mon[33169]: Reconfiguring daemon mon.a on smithi150 2024-04-05T02:44:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:37 smithi196 ceph-mon[33169]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:37 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:38.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[29558]: Reconfiguring mon.a (unknown last config time)... 2024-04-05T02:44:38.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[29558]: Reconfiguring daemon mon.a on smithi150 2024-04-05T02:44:38.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[29558]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:38.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:38.163 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[41818]: Reconfiguring mon.a (unknown last config time)... 2024-04-05T02:44:38.163 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[41818]: Reconfiguring daemon mon.a on smithi150 2024-04-05T02:44:38.163 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[41818]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:38.163 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:37 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.025 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.025 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[29558]: Reconfiguring daemon mon.c on smithi150 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:39.026 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:39.027 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:39.027 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:38 smithi150 ceph-mon[41818]: Reconfiguring daemon mon.c on smithi150 2024-04-05T02:44:39.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:44:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:38 smithi196 ceph-mon[33169]: Reconfiguring daemon mon.c on smithi150 2024-04-05T02:44:40.303 INFO:teuthology.orchestra.run.smithi150.stdout:# minimal ceph.conf for 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:44:40.303 INFO:teuthology.orchestra.run.smithi150.stdout:[global] 2024-04-05T02:44:40.304 INFO:teuthology.orchestra.run.smithi150.stdout: fsid = 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:44:40.304 INFO:teuthology.orchestra.run.smithi150.stdout: mon_host = [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] 2024-04-05T02:44:40.304 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:39 smithi150 ceph-mon[29558]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:40.304 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:39 smithi150 ceph-mon[41818]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:39 smithi196 ceph-mon[33169]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2632575226' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2632575226' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:41.113 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:41.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:41.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:41.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:41.137 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-05T02:44:41.137 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:44:41.137 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.conf 2024-04-05T02:44:41.167 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:44:41.167 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:41.168 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:40 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2632575226' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:41.168 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:41.168 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:41.168 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:44:41.168 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:44:41.169 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:41.234 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:44:41.234 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/ceph/ceph.conf 2024-04-05T02:44:41.263 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:44:41.263 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:44:41.330 INFO:tasks.cephadm:Adding mgr.y on smithi150 2024-04-05T02:44:41.330 INFO:tasks.cephadm:Adding mgr.x on smithi196 2024-04-05T02:44:41.330 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch apply mgr '2;smithi150=y;smithi196=x' 2024-04-05T02:44:42.043 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:44:42.164 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:44:42.164 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: Reconfiguring daemon mon.b on smithi196 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:42.165 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:44:42.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: Reconfiguring daemon mon.b on smithi196 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: Reconfiguring daemon mon.b on smithi196 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:42.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:42.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:42.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:42.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:43.818 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled mgr update... 2024-04-05T02:44:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:44:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:44.368 DEBUG:teuthology.orchestra.run.smithi196:mgr.x> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service 2024-04-05T02:44:44.370 INFO:tasks.cephadm:Deploying OSDs... 2024-04-05T02:44:44.371 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:44:44.371 DEBUG:teuthology.orchestra.run.smithi150:> dd if=/scratch_devs of=/dev/stdout 2024-04-05T02:44:44.390 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-05T02:44:44.390 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_1 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-05 02:44:29.194938529 +0000 2024-04-05T02:44:44.448 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-05 02:38:56.646851114 +0000 2024-04-05T02:44:44.449 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-05 02:38:56.646851114 +0000 2024-04-05T02:44:44.449 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-05 02:38:56.646851114 +0000 2024-04-05T02:44:44.449 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-05T02:44:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:44.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:44.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:44.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:44.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:44.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:44:44.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-05T02:44:44.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:44:44.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:44.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:44.481 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:44.481 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:44.481 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:44.481 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:44.481 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:44.482 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:44:44.482 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-05T02:44:44.482 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:44:44.482 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:44.511 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-05T02:44:44.511 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-05T02:44:44.512 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000289181 s, 1.8 MB/s 2024-04-05T02:44:44.513 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-05T02:44:44.573 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_2 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-05 02:44:29.194938529 +0000 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-05 02:38:56.986844182 +0000 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-05 02:38:56.986844182 +0000 2024-04-05T02:44:44.630 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-05 02:38:56.986844182 +0000 2024-04-05T02:44:44.631 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-05T02:44:44.698 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-05T02:44:44.699 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-05T02:44:44.699 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000240429 s, 2.1 MB/s 2024-04-05T02:44:44.700 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-05T02:44:44.757 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_3 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-05 02:44:29.194938529 +0000 2024-04-05T02:44:44.814 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-05 02:38:57.324837292 +0000 2024-04-05T02:44:44.815 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-05 02:38:57.324837292 +0000 2024-04-05T02:44:44.815 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-05 02:38:57.324837292 +0000 2024-04-05T02:44:44.815 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-05T02:44:44.883 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-05T02:44:44.883 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-05T02:44:44.883 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000315345 s, 1.6 MB/s 2024-04-05T02:44:44.884 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-05T02:44:44.941 DEBUG:teuthology.orchestra.run.smithi150:> stat /dev/vg_nvme/lv_4 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout:Access: 2024-04-05 02:44:29.195938509 +0000 2024-04-05T02:44:44.998 INFO:teuthology.orchestra.run.smithi150.stdout:Modify: 2024-04-05 02:38:57.655830544 +0000 2024-04-05T02:44:44.999 INFO:teuthology.orchestra.run.smithi150.stdout:Change: 2024-04-05 02:38:57.655830544 +0000 2024-04-05T02:44:44.999 INFO:teuthology.orchestra.run.smithi150.stdout: Birth: 2024-04-05 02:38:57.655830544 +0000 2024-04-05T02:44:44.999 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-05T02:44:45.065 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records in 2024-04-05T02:44:45.065 INFO:teuthology.orchestra.run.smithi150.stderr:1+0 records out 2024-04-05T02:44:45.066 INFO:teuthology.orchestra.run.smithi150.stderr:512 bytes copied, 0.000204367 s, 2.5 MB/s 2024-04-05T02:44:45.067 DEBUG:teuthology.orchestra.run.smithi150:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-05T02:44:45.124 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:44:45.124 DEBUG:teuthology.orchestra.run.smithi196:> dd if=/scratch_devs of=/dev/stdout 2024-04-05T02:44:45.140 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-05T02:44:45.140 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_1 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-04-05 02:44:20.388648381 +0000 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-04-05 02:38:56.646421260 +0000 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-04-05 02:38:56.646421260 +0000 2024-04-05T02:44:45.195 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: 2024-04-05 02:38:56.646421260 +0000 2024-04-05T02:44:45.196 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-05T02:44:45.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:44 smithi150 ceph-mon[29558]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi150=y;smithi196=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:45.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:44 smithi150 ceph-mon[29558]: Saving service mgr spec with placement smithi150=y;smithi196=x;count:2 2024-04-05T02:44:45.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:44 smithi150 ceph-mon[29558]: Deploying daemon mgr.x on smithi196 2024-04-05T02:44:45.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:45 smithi150 ceph-mon[41818]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi150=y;smithi196=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:45.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:45 smithi150 ceph-mon[41818]: Saving service mgr spec with placement smithi150=y;smithi196=x;count:2 2024-04-05T02:44:45.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:45 smithi150 ceph-mon[41818]: Deploying daemon mgr.x on smithi196 2024-04-05T02:44:45.258 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:44 smithi196 ceph-mon[33169]: from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi150=y;smithi196=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:45.258 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:44 smithi196 ceph-mon[33169]: Saving service mgr spec with placement smithi150=y;smithi196=x;count:2 2024-04-05T02:44:45.258 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:44 smithi196 ceph-mon[33169]: Deploying daemon mgr.x on smithi196 2024-04-05T02:44:45.260 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-04-05T02:44:45.260 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-04-05T02:44:45.260 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000185432 s, 2.8 MB/s 2024-04-05T02:44:45.261 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-05T02:44:45.317 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_2 2024-04-05T02:44:45.371 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-04-05 02:44:20.389648360 +0000 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-04-05 02:38:56.998413925 +0000 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-04-05 02:38:56.998413925 +0000 2024-04-05T02:44:45.372 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: 2024-04-05 02:38:56.998413925 +0000 2024-04-05T02:44:45.372 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-05T02:44:45.437 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-04-05T02:44:45.438 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-04-05T02:44:45.438 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000238137 s, 2.2 MB/s 2024-04-05T02:44:45.439 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-05T02:44:45.499 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_3 2024-04-05T02:44:45.554 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-05T02:44:45.554 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:45.554 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-04-05T02:44:45.554 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:45.555 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:45.555 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-04-05 02:44:20.389648360 +0000 2024-04-05T02:44:45.555 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-04-05 02:38:57.346406673 +0000 2024-04-05T02:44:45.555 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-04-05 02:38:57.346406673 +0000 2024-04-05T02:44:45.555 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: 2024-04-05 02:38:57.346406673 +0000 2024-04-05T02:44:45.555 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-05T02:44:45.626 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-04-05T02:44:45.626 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-04-05T02:44:45.626 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000221176 s, 2.3 MB/s 2024-04-05T02:44:45.629 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-05T02:44:45.687 DEBUG:teuthology.orchestra.run.smithi196:> stat /dev/vg_nvme/lv_4 2024-04-05T02:44:45.741 INFO:teuthology.orchestra.run.smithi196.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-05T02:44:45.741 INFO:teuthology.orchestra.run.smithi196.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-05T02:44:45.741 INFO:teuthology.orchestra.run.smithi196.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-05T02:44:45.741 INFO:teuthology.orchestra.run.smithi196.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-05T02:44:45.741 INFO:teuthology.orchestra.run.smithi196.stdout:Context: system_u:object_r:device_t:s0 2024-04-05T02:44:45.741 INFO:teuthology.orchestra.run.smithi196.stdout:Access: 2024-04-05 02:44:45.567123566 +0000 2024-04-05T02:44:45.742 INFO:teuthology.orchestra.run.smithi196.stdout:Modify: 2024-04-05 02:38:57.669399942 +0000 2024-04-05T02:44:45.742 INFO:teuthology.orchestra.run.smithi196.stdout:Change: 2024-04-05 02:38:57.669399942 +0000 2024-04-05T02:44:45.742 INFO:teuthology.orchestra.run.smithi196.stdout: Birth: 2024-04-05 02:38:57.669399942 +0000 2024-04-05T02:44:45.742 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-05T02:44:45.822 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records in 2024-04-05T02:44:45.822 INFO:teuthology.orchestra.run.smithi196.stderr:1+0 records out 2024-04-05T02:44:45.823 INFO:teuthology.orchestra.run.smithi196.stderr:512 bytes copied, 0.000247316 s, 2.1 MB/s 2024-04-05T02:44:45.824 DEBUG:teuthology.orchestra.run.smithi196:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-05T02:44:45.882 INFO:tasks.cephadm:Deploying osd.0 on smithi150 with /dev/vg_nvme/lv_4... 2024-04-05T02:44:45.882 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-05T02:44:46.142 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:46.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:45 smithi150 ceph-mon[29558]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:46.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:46 smithi150 ceph-mon[41818]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:46.329 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:45 smithi196 ceph-mon[33169]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:46.329 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:46 smithi196 ceph-mgr[35298]: -- 172.21.15.196:0/1333384551 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5632793449c0 con 0x56327931b000 2024-04-05T02:44:46.329 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:46 smithi196 bash[35260]: 5dcef717d4545091d9f2d599672352420640dc15be48fa1903dcf7b048f0f2b4 2024-04-05T02:44:46.636 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:46 smithi196 systemd[1]: Started Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:44:47.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:46.742+0000 7fb6144281c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:44:47.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:46.846+0000 7fb6144281c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:44:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:47.500 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:47.833 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:47.500+0000 7fb6144281c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:44:47.834 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:47.593+0000 7fb6144281c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:44:47.834 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:47.682+0000 7fb6144281c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:44:48.076 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-05T02:44:48.076 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-05T02:44:48.076 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-05T02:44:48.077 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-05T02:44:48.077 INFO:teuthology.orchestra.run.smithi150.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.012628 s, 830 MB/s 2024-04-05T02:44:48.077 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 2024-04-05T02:44:48.077 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-05T02:44:48.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:47.834+0000 7fb6144281c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:44:48.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:47.913+0000 7fb6144281c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:44:48.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:48.077+0000 7fb6144281c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:44:48.513 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:48.256+0000 7fb6144281c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:44:48.513 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:48.339+0000 7fb6144281c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:44:48.513 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:48.430+0000 7fb6144281c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:44:48.514 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:48.514+0000 7fb6144281c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:44:48.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: Reconfiguring mgr.y (unknown last config time)... 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:48.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:48 smithi196 ceph-mon[33169]: Reconfiguring daemon mgr.y on smithi150 2024-04-05T02:44:48.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: Reconfiguring mgr.y (unknown last config time)... 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[29558]: Reconfiguring daemon mgr.y on smithi150 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: Reconfiguring mgr.y (unknown last config time)... 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:48 smithi150 ceph-mon[41818]: Reconfiguring daemon mgr.y on smithi150 2024-04-05T02:44:49.247 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi150:vg_nvme/lv_4 2024-04-05T02:44:49.683 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:44:49.877 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-mon[33169]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:49.877 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:49.877 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:49.877 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:49.878 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:49.556+0000 7fb6144281c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:44:49.878 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:49.685+0000 7fb6144281c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:44:49.878 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:49.876+0000 7fb6144281c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:44:49.935 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[29558]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:49.935 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:49.935 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:49.935 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:49.936 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[41818]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:49.936 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:49.936 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:49.936 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:44:50.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:50.113+0000 7fb6144281c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:44:50.636 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:50.287+0000 7fb6144281c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:44:50.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:50.463+0000 7fb6144281c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:44:50.961 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:50.961 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:50.961 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:50.961 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:50.961 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:50.962 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:51.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:51.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:44:51.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:44:51.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:50.719+0000 7fb6144281c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:44:51.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:50.804+0000 7fb6144281c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:44:51.636 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:51.200+0000 7fb6144281c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:44:51.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:51 smithi196 ceph-mon[33169]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:51.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:51 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:44:51.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:51 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:44:51.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:51 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:51.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[29558]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[41818]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:44:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:51 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:44:52.257 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:51.968+0000 7fb6144281c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:44:52.257 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:52.257+0000 7fb6144281c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:44:52.750 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:52 smithi150 ceph-mon[29558]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:52.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:52 smithi150 ceph-mon[41818]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:52 smithi196 ceph-mon[33169]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:44:52.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:52.570+0000 7fb6144281c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:44:52.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:44:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:44:52.760+0000 7fb6144281c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:44:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[29558]: Standby manager daemon x started 2024-04-05T02:44:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:44:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:44:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[29558]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[41818]: Standby manager daemon x started 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:44:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:53 smithi150 ceph-mon[41818]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:54.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:53 smithi196 ceph-mon[33169]: Standby manager daemon x started 2024-04-05T02:44:54.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:53 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:44:54.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:53 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:44:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:53 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:44:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:53 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/3725031110' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:44:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:53 smithi196 ceph-mon[33169]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: mgrmap e14: y(active, since 66s), standbys: x 2024-04-05T02:44:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:44:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2445151137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]: dispatch 2024-04-05T02:44:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]: dispatch 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]': finished 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: osdmap e5: 1 total, 0 up, 1 in 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/267991900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: mgrmap e14: y(active, since 66s), standbys: x 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2445151137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]: dispatch 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]: dispatch 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]': finished 2024-04-05T02:44:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: osdmap e5: 1 total, 0 up, 1 in 2024-04-05T02:44:54.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:44:54.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:54 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/267991900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:44:55.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: mgrmap e14: y(active, since 66s), standbys: x 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2445151137' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]: dispatch 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]: dispatch 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff"}]': finished 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: osdmap e5: 1 total, 0 up, 1 in 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:44:55.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:54 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/267991900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:44:56.038 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:55 smithi196 ceph-mon[33169]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:55 smithi150 ceph-mon[29558]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:55 smithi150 ceph-mon[41818]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:57.958 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:57 smithi150 ceph-mon[29558]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:57.959 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:57 smithi150 ceph-mon[41818]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:44:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:57 smithi196 ceph-mon[33169]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:00.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:44:59 smithi150 ceph-mon[29558]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:00.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:44:59 smithi150 ceph-mon[41818]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:44:59 smithi196 ceph-mon[33169]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:01.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:45:01.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:01.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:00 smithi150 ceph-mon[41818]: Deploying daemon osd.0 on smithi150 2024-04-05T02:45:01.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:45:01.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:01.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:00 smithi150 ceph-mon[29558]: Deploying daemon osd.0 on smithi150 2024-04-05T02:45:01.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:45:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:00 smithi196 ceph-mon[33169]: Deploying daemon osd.0 on smithi150 2024-04-05T02:45:02.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:01 smithi150 ceph-mon[29558]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:02.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:01 smithi150 ceph-mon[41818]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:01 smithi196 ceph-mon[33169]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:04.129 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:03 smithi150 ceph-mon[29558]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:04.129 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:03 smithi150 ceph-mon[41818]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:04.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:03 smithi196 ceph-mon[33169]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:06.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[29558]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:06.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:06.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:06.249 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:06.249 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[41818]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:06.249 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:06.249 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:06.249 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:05 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:05 smithi196 ceph-mon[33169]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:05 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:05 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:05 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:08.117 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[29558]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:08.117 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.117 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[41818]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:08.118 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.294 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 0 on host 'smithi150' 2024-04-05T02:45:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:07 smithi196 ceph-mon[33169]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:08.835 DEBUG:teuthology.orchestra.run.smithi150:osd.0> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.0.service 2024-04-05T02:45:08.837 INFO:tasks.cephadm:Deploying osd.1 on smithi150 with /dev/vg_nvme/lv_3... 2024-04-05T02:45:08.837 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-05T02:45:09.130 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[29558]: from='osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:45:09.130 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[29558]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:45:09.130 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:09.130 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:09.130 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:09.131 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[41818]: from='osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:45:09.131 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[41818]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:45:09.131 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:09.131 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:09.131 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:08 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:09.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:08 smithi196 ceph-mon[33169]: from='osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:45:09.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:08 smithi196 ceph-mon[33169]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T02:45:09.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:08 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:09.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:08 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:09.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:08 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:09.549 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:45:10.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[29558]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[29558]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[29558]: osdmap e6: 1 total, 0 up, 1 in 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[29558]: from='osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[29558]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[41818]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[41818]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[41818]: osdmap e6: 1 total, 0 up, 1 in 2024-04-05T02:45:10.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:10.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[41818]: from='osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:10.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-mon[41818]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:10.229 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[46261]: 2024-04-05T02:45:10.015+0000 7fef12347700 -1 osd.0 0 waiting for initial osdmap 2024-04-05T02:45:10.229 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:45:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[46261]: 2024-04-05T02:45:10.017+0000 7fef0f146700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:45:10.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:10 smithi196 ceph-mon[33169]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:10 smithi196 ceph-mon[33169]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T02:45:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:10 smithi196 ceph-mon[33169]: osdmap e6: 1 total, 0 up, 1 in 2024-04-05T02:45:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:10 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:10 smithi196 ceph-mon[33169]: from='osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:10 smithi196 ceph-mon[33169]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[41818]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[41818]: osdmap e7: 1 total, 0 up, 1 in 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[29558]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[29558]: osdmap e7: 1 total, 0 up, 1 in 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:11.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:11.320 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-05T02:45:11.320 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-05T02:45:11.320 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-05T02:45:11.321 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-05T02:45:11.321 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124775 s, 840 MB/s 2024-04-05T02:45:11.321 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-05T02:45:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:11 smithi196 ceph-mon[33169]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:11 smithi196 ceph-mon[33169]: osdmap e7: 1 total, 0 up, 1 in 2024-04-05T02:45:11.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:11.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:11.923 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi150:vg_nvme/lv_3 2024-04-05T02:45:12.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:45:12.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:45:12.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[29558]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:12.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[29558]: osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696] boot 2024-04-05T02:45:12.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[29558]: osdmap e8: 1 total, 1 up, 1 in 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[41818]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[41818]: osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696] boot 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[41818]: osdmap e8: 1 total, 1 up, 1 in 2024-04-05T02:45:12.156 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:12 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:12.385 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:45:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:12 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:45:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:12 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:45:12.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:12 smithi196 ceph-mon[33169]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-05T02:45:12.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:12 smithi196 ceph-mon[33169]: osd.0 [v2:172.21.15.150:6802/4116662696,v1:172.21.15.150:6803/4116662696] boot 2024-04-05T02:45:12.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:12 smithi196 ceph-mon[33169]: osdmap e8: 1 total, 1 up, 1 in 2024-04-05T02:45:12.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:12 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:45:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:13 smithi150 ceph-mon[29558]: osdmap e9: 1 total, 1 up, 1 in 2024-04-05T02:45:13.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:13 smithi150 ceph-mon[41818]: osdmap e9: 1 total, 1 up, 1 in 2024-04-05T02:45:13.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:13 smithi196 ceph-mon[33169]: osdmap e9: 1 total, 1 up, 1 in 2024-04-05T02:45:14.290 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:14.290 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.290 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.290 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:14.290 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.290 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:14.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:14 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:14 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi150 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi150 to 16064M 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi150 2024-04-05T02:45:15.350 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi150 to 16064M 2024-04-05T02:45:15.351 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:45:15.351 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:45:15.351 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:15 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:15.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:15 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi150 2024-04-05T02:45:15.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:15 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi150 to 16064M 2024-04-05T02:45:15.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:15 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:45:15.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:15 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:45:15.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:15 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:16.225 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:16 smithi150 ceph-mon[41818]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:45:16.225 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:16 smithi150 ceph-mon[41818]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:16.225 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:16 smithi150 ceph-mon[29558]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:45:16.225 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:16 smithi150 ceph-mon[29558]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:16 smithi196 ceph-mon[33169]: from='client.24149 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:45:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:16 smithi196 ceph-mon[33169]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:18.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:18 smithi196 ceph-mon[33169]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3753230596' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1257fa7-7dac-4fed-8553-40a62778f3ec"}]: dispatch 2024-04-05T02:45:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3753230596' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1257fa7-7dac-4fed-8553-40a62778f3ec"}]': finished 2024-04-05T02:45:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:18 smithi196 ceph-mon[33169]: osdmap e10: 2 total, 1 up, 2 in 2024-04-05T02:45:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:18 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/735697297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:45:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[29558]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3753230596' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1257fa7-7dac-4fed-8553-40a62778f3ec"}]: dispatch 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3753230596' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1257fa7-7dac-4fed-8553-40a62778f3ec"}]': finished 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[29558]: osdmap e10: 2 total, 1 up, 2 in 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/735697297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[41818]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3753230596' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f1257fa7-7dac-4fed-8553-40a62778f3ec"}]: dispatch 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3753230596' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f1257fa7-7dac-4fed-8553-40a62778f3ec"}]': finished 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[41818]: osdmap e10: 2 total, 1 up, 2 in 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/735697297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:45:20.370 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:20 smithi150 ceph-mon[29558]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:20.370 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:20 smithi150 ceph-mon[41818]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:20.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:20 smithi196 ceph-mon[33169]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:22.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:22 smithi150 ceph-mon[29558]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:22.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:22 smithi150 ceph-mon[41818]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:22.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:22 smithi196 ceph-mon[33169]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[41818]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:45:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[41818]: Deploying daemon osd.1 on smithi150 2024-04-05T02:45:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[29558]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:45:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:24 smithi150 ceph-mon[29558]: Deploying daemon osd.1 on smithi150 2024-04-05T02:45:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:24 smithi196 ceph-mon[33169]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:24.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:45:24.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:24.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:24 smithi196 ceph-mon[33169]: Deploying daemon osd.1 on smithi150 2024-04-05T02:45:26.209 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:26 smithi150 ceph-mon[29558]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:26.209 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:26 smithi150 ceph-mon[41818]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:26 smithi196 ceph-mon[33169]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:28.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:28 smithi196 ceph-mon[33169]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:28 smithi150 ceph-mon[29558]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:28.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:28 smithi150 ceph-mon[41818]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:29 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:29 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:29 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:29 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:29 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:29 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:30.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:30 smithi150 ceph-mon[29558]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:30.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:30 smithi150 ceph-mon[41818]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:30.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:30 smithi196 ceph-mon[33169]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:31.346 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 1 on host 'smithi150' 2024-04-05T02:45:31.951 DEBUG:teuthology.orchestra.run.smithi150:osd.1> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1.service 2024-04-05T02:45:31.953 INFO:tasks.cephadm:Deploying osd.2 on smithi150 with /dev/vg_nvme/lv_2... 2024-04-05T02:45:31.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-05T02:45:32.262 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:45:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: from='osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:45:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:45:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: from='osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: from='osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:45:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T02:45:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:33.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[29558]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T02:45:33.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[29558]: osdmap e11: 2 total, 1 up, 2 in 2024-04-05T02:45:33.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[29558]: from='osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[29558]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[41818]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[41818]: osdmap e11: 2 total, 1 up, 2 in 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[41818]: from='osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:33.334 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-mon[41818]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:33.334 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[52666]: 2024-04-05T02:45:33.225+0000 7faf15d4b700 -1 osd.1 0 waiting for initial osdmap 2024-04-05T02:45:33.335 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:45:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[52666]: 2024-04-05T02:45:33.227+0000 7faf10345700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:45:33.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:33 smithi196 ceph-mon[33169]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T02:45:33.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:33 smithi196 ceph-mon[33169]: osdmap e11: 2 total, 1 up, 2 in 2024-04-05T02:45:33.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:33 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:33.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:33 smithi196 ceph-mon[33169]: from='osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:33.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:33 smithi196 ceph-mon[33169]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[29558]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[29558]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[29558]: osdmap e12: 2 total, 1 up, 2 in 2024-04-05T02:45:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[41818]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[41818]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[41818]: osdmap e12: 2 total, 1 up, 2 in 2024-04-05T02:45:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:34 smithi196 ceph-mon[33169]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:34 smithi196 ceph-mon[33169]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:34 smithi196 ceph-mon[33169]: osdmap e12: 2 total, 1 up, 2 in 2024-04-05T02:45:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[41818]: osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304] boot 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[41818]: osdmap e13: 2 total, 2 up, 2 in 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:45:35.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:45:35.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[29558]: osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304] boot 2024-04-05T02:45:35.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[29558]: osdmap e13: 2 total, 2 up, 2 in 2024-04-05T02:45:35.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:35 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:35 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:45:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:35 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:45:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:35 smithi196 ceph-mon[33169]: osd.1 [v2:172.21.15.150:6810/3820311304,v1:172.21.15.150:6811/3820311304] boot 2024-04-05T02:45:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:35 smithi196 ceph-mon[33169]: osdmap e13: 2 total, 2 up, 2 in 2024-04-05T02:45:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:35 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:45:35.666 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-05T02:45:35.667 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-05T02:45:35.667 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-05T02:45:35.667 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-05T02:45:35.667 INFO:teuthology.orchestra.run.smithi150.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125207 s, 837 MB/s 2024-04-05T02:45:35.667 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-05T02:45:36.428 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:36.428 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: osdmap e14: 2 total, 2 up, 2 in 2024-04-05T02:45:36.428 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.428 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.428 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: osdmap e14: 2 total, 2 up, 2 in 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:36.429 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:36.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:36.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:36 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: osdmap e14: 2 total, 2 up, 2 in 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:36.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:36 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:36.729 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi150:vg_nvme/lv_2 2024-04-05T02:45:36.982 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:45:37.354 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi150 2024-04-05T02:45:37.354 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi150 to 8032M 2024-04-05T02:45:37.354 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:37.354 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi150 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi150 to 8032M 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:37.355 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:37 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:37 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi150 2024-04-05T02:45:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:37 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi150 to 8032M 2024-04-05T02:45:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:37 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:37 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:37 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:37 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:38.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:38 smithi150 ceph-mon[29558]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:38.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:38 smithi150 ceph-mon[41818]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:38 smithi196 ceph-mon[33169]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:39.282 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:39 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:45:39.282 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:39 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:45:39.283 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:39 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:39.283 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:39 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:45:39.283 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:39 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:45:39.283 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:39 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:39 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:45:39.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:39 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:45:39.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:39 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:40.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:40 smithi150 ceph-mon[29558]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:45:40.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:40 smithi150 ceph-mon[29558]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:40.356 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:40 smithi150 ceph-mon[41818]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:45:40.356 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:40 smithi150 ceph-mon[41818]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:40.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:40 smithi196 ceph-mon[33169]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:45:40.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:40 smithi196 ceph-mon[33169]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:41 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1458769107' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]: dispatch 2024-04-05T02:45:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:41 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]: dispatch 2024-04-05T02:45:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:41 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]': finished 2024-04-05T02:45:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:41 smithi196 ceph-mon[33169]: osdmap e15: 3 total, 2 up, 3 in 2024-04-05T02:45:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:41 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:41.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1458769107' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]: dispatch 2024-04-05T02:45:41.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]: dispatch 2024-04-05T02:45:41.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]': finished 2024-04-05T02:45:41.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[41818]: osdmap e15: 3 total, 2 up, 3 in 2024-04-05T02:45:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1458769107' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]: dispatch 2024-04-05T02:45:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]: dispatch 2024-04-05T02:45:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c672159-ed53-4d76-acd4-f29bbdb1483f"}]': finished 2024-04-05T02:45:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[29558]: osdmap e15: 3 total, 2 up, 3 in 2024-04-05T02:45:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:41 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:42.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:42 smithi196 ceph-mon[33169]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:42.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:42 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1617557411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:45:42.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:42 smithi150 ceph-mon[29558]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:42.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:42 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1617557411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:45:42.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:42 smithi150 ceph-mon[41818]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:42.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:42 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1617557411' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:45:44.436 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:44 smithi150 ceph-mon[29558]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:44.437 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:44 smithi150 ceph-mon[41818]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:44.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:44 smithi196 ceph-mon[33169]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:46.345 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:46 smithi150 ceph-mon[29558]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:46.345 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:46 smithi150 ceph-mon[41818]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:46.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:46 smithi196 ceph-mon[33169]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:47.539 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:45:47.539 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:47 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:47.540 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:45:47.540 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:47 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:47.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:45:47.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:47 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:48 smithi150 ceph-mon[29558]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:48 smithi150 ceph-mon[29558]: Deploying daemon osd.2 on smithi150 2024-04-05T02:45:48.538 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:48 smithi150 ceph-mon[41818]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:48.538 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:48 smithi150 ceph-mon[41818]: Deploying daemon osd.2 on smithi150 2024-04-05T02:45:48.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:48 smithi196 ceph-mon[33169]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:48.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:48 smithi196 ceph-mon[33169]: Deploying daemon osd.2 on smithi150 2024-04-05T02:45:50.541 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:50 smithi150 ceph-mon[29558]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:50.541 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:50 smithi150 ceph-mon[41818]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:50 smithi196 ceph-mon[33169]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:52.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[29558]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:52.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:52.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:52.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:52.577 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[41818]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:52.577 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:52.577 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:52.577 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:52 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:52 smithi196 ceph-mon[33169]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:52 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:52 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:52 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:54.610 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:54 smithi150 ceph-mon[29558]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:54.611 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:54 smithi150 ceph-mon[41818]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:54 smithi196 ceph-mon[33169]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[41818]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:45:55.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:55 smithi150 ceph-mon[29558]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:45:55.662 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 2 on host 'smithi150' 2024-04-05T02:45:55.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:45:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:55 smithi196 ceph-mon[33169]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T02:45:56.285 DEBUG:teuthology.orchestra.run.smithi150:osd.2> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.2.service 2024-04-05T02:45:56.287 INFO:tasks.cephadm:Deploying osd.3 on smithi150 with /dev/vg_nvme/lv_1... 2024-04-05T02:45:56.287 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-05T02:45:56.830 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:56.830 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:56.830 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:56.830 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T02:45:56.830 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: osdmap e16: 3 total, 2 up, 3 in 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[41818]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: osdmap e16: 3 total, 2 up, 3 in 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:56.831 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-mon[29558]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: osdmap e16: 3 total, 2 up, 3 in 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:56 smithi196 ceph-mon[33169]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:45:57.081 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:45:57.174 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[58685]: 2024-04-05T02:45:56.906+0000 7f53f16f5700 -1 osd.2 0 waiting for initial osdmap 2024-04-05T02:45:57.174 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:45:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[58685]: 2024-04-05T02:45:56.909+0000 7f53ee4f4700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[29558]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[29558]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[29558]: osdmap e17: 3 total, 2 up, 3 in 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[41818]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[41818]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[41818]: osdmap e17: 3 total, 2 up, 3 in 2024-04-05T02:45:57.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:57.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:57 smithi196 ceph-mon[33169]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:45:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:57 smithi196 ceph-mon[33169]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:45:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:57 smithi196 ceph-mon[33169]: osdmap e17: 3 total, 2 up, 3 in 2024-04-05T02:45:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:45:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:45:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[29558]: osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618] boot 2024-04-05T02:45:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[29558]: osdmap e18: 3 total, 3 up, 3 in 2024-04-05T02:45:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:45:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:45:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[41818]: osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618] boot 2024-04-05T02:45:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[41818]: osdmap e18: 3 total, 3 up, 3 in 2024-04-05T02:45:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:58 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:59.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:58 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:45:59.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:58 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:45:59.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:58 smithi196 ceph-mon[33169]: osd.2 [v2:172.21.15.150:6818/3881763618,v1:172.21.15.150:6819/3881763618] boot 2024-04-05T02:45:59.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:58 smithi196 ceph-mon[33169]: osdmap e18: 3 total, 3 up, 3 in 2024-04-05T02:45:59.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:58 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:45:59.979 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-05T02:45:59.979 INFO:teuthology.orchestra.run.smithi150.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-05T02:45:59.979 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10+0 records in 2024-04-05T02:45:59.979 INFO:teuthology.orchestra.run.smithi150.stderr:10+0 records out 2024-04-05T02:45:59.979 INFO:teuthology.orchestra.run.smithi150.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133294 s, 787 MB/s 2024-04-05T02:45:59.979 INFO:teuthology.orchestra.run.smithi150.stderr:--> Zapping successful for: 2024-04-05T02:46:00.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:59 smithi150 ceph-mon[29558]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:46:00.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:59 smithi150 ceph-mon[29558]: osdmap e19: 3 total, 3 up, 3 in 2024-04-05T02:46:00.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:45:59 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-05T02:46:00.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:59 smithi150 ceph-mon[41818]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:46:00.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:59 smithi150 ceph-mon[41818]: osdmap e19: 3 total, 3 up, 3 in 2024-04-05T02:46:00.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:45:59 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-05T02:46:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:59 smithi196 ceph-mon[33169]: pgmap v74: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-05T02:46:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:59 smithi196 ceph-mon[33169]: osdmap e19: 3 total, 3 up, 3 in 2024-04-05T02:46:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:45:59 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-05T02:46:00.686 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi150:vg_nvme/lv_1 2024-04-05T02:46:00.941 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:46:01.201 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-05T02:46:01.201 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: osdmap e20: 3 total, 3 up, 3 in 2024-04-05T02:46:01.201 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:46:01.201 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi150 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi150 to 5354M 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-05T02:46:01.202 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: osdmap e20: 3 total, 3 up, 3 in 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi150 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi150 to 5354M 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:01.203 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:00 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: osdmap e20: 3 total, 3 up, 3 in 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi150 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi150 to 5354M 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:01.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:00 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:01.477 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63143]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:46:01.477 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63143]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:01.477 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63143]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:01.477 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63143]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:01.888 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63146]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:46:01.889 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63146]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:01.889 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63146]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:01.889 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63146]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:02.159 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63155]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:46:02.159 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63155]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:02.159 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63155]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:02.159 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:01 smithi150 sudo[63155]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:02.159 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[29558]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:02.159 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:46:02.159 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[29558]: osdmap e21: 3 total, 3 up, 3 in 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[29558]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[41818]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[41818]: osdmap e21: 3 total, 3 up, 3 in 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[41818]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:46:02.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:01 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:01 smithi196 ceph-mon[33169]: pgmap v77: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:01 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:46:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:01 smithi196 ceph-mon[33169]: osdmap e21: 3 total, 3 up, 3 in 2024-04-05T02:46:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:01 smithi196 ceph-mon[33169]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:46:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:01 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:02.412 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63178]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:46:02.412 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63178]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:02.412 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63178]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:02.412 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63178]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:02.675 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63201]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:46:02.675 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63201]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:02.675 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63201]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:02.675 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63201]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:02.936 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63235]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:46:02.936 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63235]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:02.936 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63235]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:02.936 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63235]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:02.937 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63231]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:46:02.937 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63231]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:02.937 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63231]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:02.937 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 02:46:02 smithi150 sudo[63231]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:03.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:03 smithi150 ceph-mon[41818]: osdmap e22: 3 total, 3 up, 3 in 2024-04-05T02:46:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63238]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:46:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63238]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63238]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:03.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63238]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:03.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:03 smithi150 ceph-mon[29558]: osdmap e22: 3 total, 3 up, 3 in 2024-04-05T02:46:03.441 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 ceph-mon[33169]: osdmap e22: 3 total, 3 up, 3 in 2024-04-05T02:46:03.441 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36009]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:46:03.442 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36009]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:03.442 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36009]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:03.442 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36009]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36012]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:46:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36012]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36012]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:03 smithi196 sudo[36012]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:04.210 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63403]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-05T02:46:04.210 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63403]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:04.210 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63403]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:04.210 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:03 smithi150 sudo[63403]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:04.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 sudo[63414]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 sudo[63414]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 sudo[63414]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 sudo[63414]: pam_unix(sudo:session): session closed for user root 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi150:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:04.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:04.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:04.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:46:04.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:46:04.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:46:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:05 smithi196 ceph-mon[33169]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:05 smithi196 ceph-mon[33169]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:46:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:05 smithi196 ceph-mon[33169]: Cluster is now healthy 2024-04-05T02:46:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:05 smithi196 ceph-mon[33169]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:05 smithi196 ceph-mon[33169]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-05T02:46:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[29558]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[29558]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[29558]: Cluster is now healthy 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[29558]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[29558]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[41818]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[41818]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[41818]: Cluster is now healthy 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[41818]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-05T02:46:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:05 smithi150 ceph-mon[41818]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-05T02:46:06.629 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:06.629 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3859513545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]: dispatch 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]: dispatch 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]': finished 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: osdmap e23: 4 total, 3 up, 4 in 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/618164151' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3859513545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]: dispatch 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]: dispatch 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]': finished 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: osdmap e23: 4 total, 3 up, 4 in 2024-04-05T02:46:06.630 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:06.631 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:06 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/618164151' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: pgmap v81: 1 pgs: 1 creating+peering; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3859513545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]: dispatch 2024-04-05T02:46:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]: dispatch 2024-04-05T02:46:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bac5c8a2-380b-4767-8305-79febfebf621"}]': finished 2024-04-05T02:46:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: osdmap e23: 4 total, 3 up, 4 in 2024-04-05T02:46:06.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:06.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:06 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/618164151' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:08.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:08 smithi196 ceph-mon[33169]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:08 smithi150 ceph-mon[29558]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:08 smithi150 ceph-mon[41818]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:10.633 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:10 smithi150 ceph-mon[29558]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:10.634 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:10 smithi150 ceph-mon[41818]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:10.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:10 smithi196 ceph-mon[33169]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:12.521 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:12 smithi150 ceph-mon[41818]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:12.521 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:12 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:46:12.521 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:12 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:12.521 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:12 smithi150 ceph-mon[29558]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:12.521 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:12 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:46:12.521 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:12 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:12.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:12 smithi196 ceph-mon[33169]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:12.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:12 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:46:12.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:12 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:13.459 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:13 smithi150 ceph-mon[29558]: Deploying daemon osd.3 on smithi150 2024-04-05T02:46:13.459 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:13 smithi150 ceph-mon[41818]: Deploying daemon osd.3 on smithi150 2024-04-05T02:46:13.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:13 smithi196 ceph-mon[33169]: Deploying daemon osd.3 on smithi150 2024-04-05T02:46:14.459 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:14 smithi150 ceph-mon[29558]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:14.459 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:14 smithi150 ceph-mon[41818]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:14 smithi196 ceph-mon[33169]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:16.400 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:16 smithi150 ceph-mon[41818]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:16.401 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:16 smithi150 ceph-mon[29558]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:16.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:16 smithi196 ceph-mon[33169]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:17.507 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:17 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:17.507 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:17 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:17.507 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:17 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:17.507 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:17 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:17.508 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:17 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:17.508 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:17 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:17 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:17 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:17.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:17 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:18 smithi150 ceph-mon[29558]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:18.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:18 smithi150 ceph-mon[41818]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:18 smithi196 ceph-mon[33169]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:46:20.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[29558]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:46:20.431 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:20 smithi150 ceph-mon[41818]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:46:20.485 INFO:teuthology.orchestra.run.smithi150.stdout:Created osd(s) 3 on host 'smithi150' 2024-04-05T02:46:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:46:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:20 smithi196 ceph-mon[33169]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T02:46:21.141 DEBUG:teuthology.orchestra.run.smithi150:osd.3> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.3.service 2024-04-05T02:46:21.143 INFO:tasks.cephadm:Deploying osd.4 on smithi196 with /dev/vg_nvme/lv_4... 2024-04-05T02:46:21.144 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-05T02:46:21.399 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: osdmap e24: 4 total, 3 up, 4 in 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[29558]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:21.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: osdmap e24: 4 total, 3 up, 4 in 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:46:21.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-mon[41818]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:21.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:21.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:21.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:21.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T02:46:21.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: osdmap e24: 4 total, 3 up, 4 in 2024-04-05T02:46:21.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:21.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:46:21.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T02:46:21.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:21 smithi196 ceph-mon[33169]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:22.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[65322]: 2024-04-05T02:46:21.810+0000 7f705859b700 -1 osd.3 0 waiting for initial osdmap 2024-04-05T02:46:22.228 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 02:46:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[65322]: 2024-04-05T02:46:21.814+0000 7f7052b95700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:46:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:22 smithi196 ceph-mon[33169]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:46:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:22 smithi196 ceph-mon[33169]: osdmap e25: 4 total, 3 up, 4 in 2024-04-05T02:46:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:22 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:22 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[29558]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[29558]: osdmap e25: 4 total, 3 up, 4 in 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[41818]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]': finished 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[41818]: osdmap e25: 4 total, 3 up, 4 in 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:22 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.336 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-05T02:46:23.337 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-05T02:46:23.337 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-04-05T02:46:23.337 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-04-05T02:46:23.337 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.011213 s, 935 MB/s 2024-04-05T02:46:23.338 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[29558]: osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788] boot 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[29558]: osdmap e26: 4 total, 4 up, 4 in 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[29558]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:46:23.948 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[41818]: osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788] boot 2024-04-05T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[41818]: osdmap e26: 4 total, 4 up, 4 in 2024-04-05T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.949 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:23 smithi150 ceph-mon[41818]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:23.950 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:23 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:46:23.950 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:23 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:46:23.950 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:23 smithi196 ceph-mon[33169]: osd.3 [v2:172.21.15.150:6826/2956472788,v1:172.21.15.150:6827/2956472788] boot 2024-04-05T02:46:23.950 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:23 smithi196 ceph-mon[33169]: osdmap e26: 4 total, 4 up, 4 in 2024-04-05T02:46:23.950 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:23 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:46:23.950 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:23 smithi196 ceph-mon[33169]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-05T02:46:23.976 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi196:vg_nvme/lv_4 2024-04-05T02:46:24.227 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:46:24.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: osdmap e27: 4 total, 4 up, 4 in 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi150 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi150 to 4016M 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:24 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: osdmap e27: 4 total, 4 up, 4 in 2024-04-05T02:46:25.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi150 2024-04-05T02:46:25.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi150 to 4016M 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: osdmap e27: 4 total, 4 up, 4 in 2024-04-05T02:46:25.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi150 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi150 to 4016M 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:25.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:24 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:26.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:25 smithi196 ceph-mon[33169]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:25 smithi150 ceph-mon[29558]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:25 smithi150 ceph-mon[41818]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:26.981 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:26 smithi196 ceph-mon[33169]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:26.982 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:26.982 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:26.982 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:26 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:27.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[41818]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:27.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:27.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:27.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[29558]: from='client.24208 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:26 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:28.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:27 smithi196 ceph-mon[33169]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:28.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:27 smithi150 ceph-mon[29558]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:27 smithi150 ceph-mon[41818]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:28 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/113927560' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]: dispatch 2024-04-05T02:46:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:28 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]: dispatch 2024-04-05T02:46:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:28 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]': finished 2024-04-05T02:46:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:28 smithi196 ceph-mon[33169]: osdmap e28: 5 total, 4 up, 5 in 2024-04-05T02:46:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:28 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/113927560' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]: dispatch 2024-04-05T02:46:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]: dispatch 2024-04-05T02:46:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]': finished 2024-04-05T02:46:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[29558]: osdmap e28: 5 total, 4 up, 5 in 2024-04-05T02:46:29.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/113927560' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]: dispatch 2024-04-05T02:46:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]: dispatch 2024-04-05T02:46:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "361bfd73-2a77-4096-be7e-37c137d35142"}]': finished 2024-04-05T02:46:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[41818]: osdmap e28: 5 total, 4 up, 5 in 2024-04-05T02:46:29.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:28 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:30.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:29 smithi196 ceph-mon[33169]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:30.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:29 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/1389257021' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:29 smithi150 ceph-mon[29558]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:29 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/1389257021' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:29 smithi150 ceph-mon[41818]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/1389257021' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:32.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:31 smithi196 ceph-mon[33169]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:31 smithi150 ceph-mon[29558]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:32.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:31 smithi150 ceph-mon[41818]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:34.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:33 smithi150 ceph-mon[41818]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:34.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:33 smithi150 ceph-mon[29558]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:34.294 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:33 smithi196 ceph-mon[33169]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:35.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:46:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:46:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:35.272 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:46:35.273 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:36.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:35 smithi196 ceph-mon[33169]: Deploying daemon osd.4 on smithi196 2024-04-05T02:46:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:35 smithi196 ceph-mon[33169]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:36.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:35 smithi150 ceph-mon[41818]: Deploying daemon osd.4 on smithi196 2024-04-05T02:46:36.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:35 smithi150 ceph-mon[41818]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:35 smithi150 ceph-mon[29558]: Deploying daemon osd.4 on smithi196 2024-04-05T02:46:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:35 smithi150 ceph-mon[29558]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:38.127 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:37 smithi196 ceph-mon[33169]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:37 smithi150 ceph-mon[29558]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:38.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:37 smithi150 ceph-mon[41818]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:40.218 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:39 smithi196 ceph-mon[33169]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:40.218 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:39 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:40.219 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:39 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:40.219 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:39 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:40.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[41818]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:40.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:40.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:40.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[29558]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:39 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:41 smithi150 ceph-mon[29558]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:42.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:41 smithi150 ceph-mon[41818]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:42.316 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:41 smithi196 ceph-mon[33169]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:42.637 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 4 on host 'smithi196' 2024-04-05T02:46:43.244 DEBUG:teuthology.orchestra.run.smithi196:osd.4> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.4.service 2024-04-05T02:46:43.247 INFO:tasks.cephadm:Deploying osd.5 on smithi196 with /dev/vg_nvme/lv_3... 2024-04-05T02:46:43.247 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-05T02:46:43.554 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:46:43.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: from='osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:46:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:46:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:43 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: from='osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: from='osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:46:43.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-05T02:46:43.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:43.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:43.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:43 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:44.709 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-mon[33169]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-05T02:46:44.710 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-mon[33169]: osdmap e29: 5 total, 4 up, 5 in 2024-04-05T02:46:44.710 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:44.710 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-mon[33169]: from='osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:46:44.710 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-mon[33169]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:46:44.710 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T02:46:44.657+0000 7f7a9b13f700 -1 osd.4 0 waiting for initial osdmap 2024-04-05T02:46:44.710 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 02:46:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T02:46:44.662+0000 7f7a94f38700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:46:44.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[29558]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[29558]: osdmap e29: 5 total, 4 up, 5 in 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[29558]: from='osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[29558]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[41818]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[41818]: osdmap e29: 5 total, 4 up, 5 in 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[41818]: from='osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:46:44.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:44 smithi150 ceph-mon[41818]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: osdmap e30: 5 total, 4 up, 5 in 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274] boot 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: osdmap e31: 5 total, 5 up, 5 in 2024-04-05T02:46:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:45 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.899 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-05T02:46:45.899 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-05T02:46:45.899 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-04-05T02:46:45.899 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-04-05T02:46:45.899 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124347 s, 843 MB/s 2024-04-05T02:46:45.899 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-04-05T02:46:45.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:46:45.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: osdmap e30: 5 total, 4 up, 5 in 2024-04-05T02:46:45.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274] boot 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: osdmap e31: 5 total, 5 up, 5 in 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: osdmap e30: 5 total, 4 up, 5 in 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-05T02:46:45.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: osd.4 [v2:172.21.15.196:6800/929114274,v1:172.21.15.196:6801/929114274] boot 2024-04-05T02:46:45.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: osdmap e31: 5 total, 5 up, 5 in 2024-04-05T02:46:45.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:45 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:46:46.790 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:46 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:46:46.790 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:46 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:46:46.790 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:46 smithi196 ceph-mon[33169]: osdmap e32: 5 total, 5 up, 5 in 2024-04-05T02:46:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:46 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:46:46.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:46 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:46:46.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:46 smithi150 ceph-mon[29558]: osdmap e32: 5 total, 5 up, 5 in 2024-04-05T02:46:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:46 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:46:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:46 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:46:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:46 smithi150 ceph-mon[41818]: osdmap e32: 5 total, 5 up, 5 in 2024-04-05T02:46:47.309 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi196:vg_nvme/lv_3 2024-04-05T02:46:47.799 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:46:47.924 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:47 smithi196 ceph-mon[33169]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:46:47.924 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:47 smithi196 ceph-mon[33169]: osdmap e33: 5 total, 5 up, 5 in 2024-04-05T02:46:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:47 smithi150 ceph-mon[29558]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:46:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:47 smithi150 ceph-mon[29558]: osdmap e33: 5 total, 5 up, 5 in 2024-04-05T02:46:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:47 smithi150 ceph-mon[41818]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:46:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:47 smithi150 ceph-mon[41818]: osdmap e33: 5 total, 5 up, 5 in 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: osdmap e34: 5 total, 5 up, 5 in 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi196 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi196 to 17088M 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.336 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:49.337 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:49.337 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.337 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:49.337 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:49.337 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:49.337 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:49 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: osdmap e34: 5 total, 5 up, 5 in 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi196 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi196 to 17088M 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: osdmap e34: 5 total, 5 up, 5 in 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi196 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi196 to 17088M 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:49.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:49.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:49.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:46:49.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:49.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:46:49.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:49 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:46:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[29558]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:46:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[29558]: osdmap e35: 5 total, 5 up, 5 in 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[41818]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[41818]: osdmap e35: 5 total, 5 up, 5 in 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:50.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:50.488 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:50 smithi196 ceph-mon[33169]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:46:50.488 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:50 smithi196 ceph-mon[33169]: osdmap e35: 5 total, 5 up, 5 in 2024-04-05T02:46:50.488 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:46:50.488 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:46:50.489 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:51 smithi150 ceph-mon[29558]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:51.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:51 smithi150 ceph-mon[41818]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:51.486 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:51 smithi196 ceph-mon[33169]: from='client.24235 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:46:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:52 smithi150 ceph-mon[29558]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 136 KiB/s, 0 objects/s recovering 2024-04-05T02:46:52.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:52 smithi150 ceph-mon[41818]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 136 KiB/s, 0 objects/s recovering 2024-04-05T02:46:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:52 smithi196 ceph-mon[33169]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 136 KiB/s, 0 objects/s recovering 2024-04-05T02:46:53.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/1904324178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]: dispatch 2024-04-05T02:46:53.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]: dispatch 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]': finished 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[41818]: osdmap e36: 6 total, 5 up, 6 in 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/1904324178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]: dispatch 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]: dispatch 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]': finished 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[29558]: osdmap e36: 6 total, 5 up, 6 in 2024-04-05T02:46:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:53 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:46:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:53 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/1904324178' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]: dispatch 2024-04-05T02:46:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:53 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]: dispatch 2024-04-05T02:46:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:53 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "845873d4-e80b-4f74-b331-5f09131f995f"}]': finished 2024-04-05T02:46:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:53 smithi196 ceph-mon[33169]: osdmap e36: 6 total, 5 up, 6 in 2024-04-05T02:46:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:53 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:46:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:54 smithi150 ceph-mon[41818]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-04-05T02:46:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:54 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/2714500586' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:54.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:54 smithi150 ceph-mon[29558]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-04-05T02:46:54.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:54 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/2714500586' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:54 smithi196 ceph-mon[33169]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-04-05T02:46:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:54 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/2714500586' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:46:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:56 smithi196 ceph-mon[33169]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-05T02:46:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:56 smithi150 ceph-mon[41818]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-05T02:46:56.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:56 smithi150 ceph-mon[29558]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-05T02:46:58.467 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:58 smithi196 ceph-mon[33169]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:46:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:58 smithi150 ceph-mon[41818]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:46:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:58 smithi150 ceph-mon[29558]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:46:59.428 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:59 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T02:46:59.428 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:46:59 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:59.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:59 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T02:46:59.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:46:59 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:46:59.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:59 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T02:46:59.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:46:59 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:00.441 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:00 smithi196 ceph-mon[33169]: Deploying daemon osd.5 on smithi196 2024-04-05T02:47:00.441 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:00 smithi196 ceph-mon[33169]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-05T02:47:00.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:00 smithi150 ceph-mon[29558]: Deploying daemon osd.5 on smithi196 2024-04-05T02:47:00.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:00 smithi150 ceph-mon[29558]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-05T02:47:00.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:00 smithi150 ceph-mon[41818]: Deploying daemon osd.5 on smithi196 2024-04-05T02:47:00.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:00 smithi150 ceph-mon[41818]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-05T02:47:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:02 smithi196 ceph-mon[33169]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:02 smithi150 ceph-mon[29558]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:02.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:02 smithi150 ceph-mon[41818]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:04.504 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:04 smithi196 ceph-mon[33169]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:04.504 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:04.505 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:04.505 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:04 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[41818]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:04.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[29558]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:04.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:04.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:04.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:04 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:06.501 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:06 smithi196 ceph-mon[33169]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:06.530 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:06 smithi150 ceph-mon[41818]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:06.530 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:06 smithi150 ceph-mon[29558]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:07.595 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 5 on host 'smithi196' 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:07 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:07 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:08.173 DEBUG:teuthology.orchestra.run.smithi196:osd.5> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.5.service 2024-04-05T02:47:08.176 INFO:tasks.cephadm:Deploying osd.6 on smithi196 with /dev/vg_nvme/lv_2... 2024-04-05T02:47:08.176 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-05T02:47:08.735 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-mon[33169]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-05T02:47:08.735 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-mon[33169]: osdmap e37: 6 total, 5 up, 6 in 2024-04-05T02:47:08.735 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:08.736 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-mon[33169]: from='osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:08.736 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-mon[33169]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:08.736 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T02:47:08.711+0000 7fde9f714700 -1 osd.5 0 waiting for initial osdmap 2024-04-05T02:47:08.736 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 02:47:08 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T02:47:08.716+0000 7fde9c513700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:47:08.966 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:47:08.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[41818]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-05T02:47:08.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[41818]: osdmap e37: 6 total, 5 up, 6 in 2024-04-05T02:47:08.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[41818]: from='osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[41818]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[29558]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[29558]: osdmap e37: 6 total, 5 up, 6 in 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[29558]: from='osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:08 smithi150 ceph-mon[29558]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: osdmap e38: 6 total, 5 up, 6 in 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894] boot 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: osdmap e39: 6 total, 6 up, 6 in 2024-04-05T02:47:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:47:09.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: osdmap e38: 6 total, 5 up, 6 in 2024-04-05T02:47:09.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894] boot 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: osdmap e39: 6 total, 6 up, 6 in 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: osdmap e38: 6 total, 5 up, 6 in 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-05T02:47:09.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: osd.5 [v2:172.21.15.196:6808/2903834894,v1:172.21.15.196:6809/2903834894] boot 2024-04-05T02:47:09.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: osdmap e39: 6 total, 6 up, 6 in 2024-04-05T02:47:09.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:47:10.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:10 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:47:10.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:10 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:47:10.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:10 smithi196 ceph-mon[33169]: osdmap e40: 6 total, 6 up, 6 in 2024-04-05T02:47:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:10 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:47:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:10 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:47:10.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:10 smithi150 ceph-mon[29558]: osdmap e40: 6 total, 6 up, 6 in 2024-04-05T02:47:10.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:10 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:47:10.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:10 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:47:10.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:10 smithi150 ceph-mon[41818]: osdmap e40: 6 total, 6 up, 6 in 2024-04-05T02:47:11.838 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:11 smithi196 ceph-mon[33169]: osdmap e41: 6 total, 6 up, 6 in 2024-04-05T02:47:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[29558]: osdmap e41: 6 total, 6 up, 6 in 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:11.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:11 smithi150 ceph-mon[41818]: osdmap e41: 6 total, 6 up, 6 in 2024-04-05T02:47:12.276 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-05T02:47:12.277 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-05T02:47:12.277 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-04-05T02:47:12.277 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-04-05T02:47:12.277 INFO:teuthology.orchestra.run.smithi196.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113184 s, 926 MB/s 2024-04-05T02:47:12.277 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 2024-04-05T02:47:12.277 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-04-05T02:47:12.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:12 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi196 2024-04-05T02:47:12.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:12 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi196 to 8544M 2024-04-05T02:47:12.939 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi196:vg_nvme/lv_2 2024-04-05T02:47:13.188 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:47:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:12 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi196 2024-04-05T02:47:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:12 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi196 to 8544M 2024-04-05T02:47:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:12 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi196 2024-04-05T02:47:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:12 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi196 to 8544M 2024-04-05T02:47:13.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:13 smithi196 ceph-mon[33169]: osdmap e42: 6 total, 6 up, 6 in 2024-04-05T02:47:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:13 smithi196 ceph-mon[33169]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:13 smithi196 ceph-mon[33169]: osdmap e43: 6 total, 6 up, 6 in 2024-04-05T02:47:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:13 smithi150 ceph-mon[41818]: osdmap e42: 6 total, 6 up, 6 in 2024-04-05T02:47:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:13 smithi150 ceph-mon[41818]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:13 smithi150 ceph-mon[41818]: osdmap e43: 6 total, 6 up, 6 in 2024-04-05T02:47:14.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:13 smithi150 ceph-mon[29558]: osdmap e42: 6 total, 6 up, 6 in 2024-04-05T02:47:14.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:13 smithi150 ceph-mon[29558]: pgmap v135: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:14.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:13 smithi150 ceph-mon[29558]: osdmap e43: 6 total, 6 up, 6 in 2024-04-05T02:47:16.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[29558]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[29558]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[41818]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[41818]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:47:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:15 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:15 smithi196 ceph-mon[33169]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:15 smithi196 ceph-mon[33169]: from='client.24262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:47:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:15 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:47:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:15 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:47:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:15 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[41818]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/2199322485' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]: dispatch 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]: dispatch 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]': finished 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[41818]: osdmap e44: 7 total, 6 up, 7 in 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[29558]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/2199322485' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]: dispatch 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]: dispatch 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]': finished 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[29558]: osdmap e44: 7 total, 6 up, 7 in 2024-04-05T02:47:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:17 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:17 smithi196 ceph-mon[33169]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:17 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/2199322485' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]: dispatch 2024-04-05T02:47:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:17 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]: dispatch 2024-04-05T02:47:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:17 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc"}]': finished 2024-04-05T02:47:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:17 smithi196 ceph-mon[33169]: osdmap e44: 7 total, 6 up, 7 in 2024-04-05T02:47:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:17 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:19.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/3545005758' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:47:19.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/3545005758' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:47:19.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/3545005758' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:47:20.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:19 smithi150 ceph-mon[41818]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:20.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:19 smithi150 ceph-mon[29558]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:20.348 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:19 smithi196 ceph-mon[33169]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:22.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:21 smithi150 ceph-mon[41818]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:22.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:21 smithi150 ceph-mon[29558]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:22.353 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:21 smithi196 ceph-mon[33169]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:24.216 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:23 smithi196 ceph-mon[33169]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:24.216 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:23 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T02:47:24.216 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:23 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:24.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:23 smithi150 ceph-mon[29558]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:24.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:23 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T02:47:24.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:23 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:23 smithi150 ceph-mon[41818]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:23 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T02:47:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:23 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:25.091 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:24 smithi196 ceph-mon[33169]: Deploying daemon osd.6 on smithi196 2024-04-05T02:47:25.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:24 smithi150 ceph-mon[29558]: Deploying daemon osd.6 on smithi196 2024-04-05T02:47:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:24 smithi150 ceph-mon[41818]: Deploying daemon osd.6 on smithi196 2024-04-05T02:47:26.056 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:25 smithi196 ceph-mon[33169]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:25 smithi150 ceph-mon[29558]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:25 smithi150 ceph-mon[41818]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:28.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:27 smithi150 ceph-mon[29558]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:27 smithi150 ceph-mon[41818]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:28.255 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:27 smithi196 ceph-mon[33169]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[29558]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:30.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:30.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[41818]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:29 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:30.330 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:29 smithi196 ceph-mon[33169]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:30.330 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:29 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:30.330 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:29 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:30.330 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:29 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:32.208 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:31 smithi196 ceph-mon[33169]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:31 smithi150 ceph-mon[29558]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:31 smithi150 ceph-mon[41818]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:32.292 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 6 on host 'smithi196' 2024-04-05T02:47:32.925 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:32 smithi196 ceph-mon[33169]: from='osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T02:47:32.925 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:32 smithi196 ceph-mon[33169]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T02:47:32.925 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:32.925 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:32.925 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:32.925 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:32 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:32.927 DEBUG:teuthology.orchestra.run.smithi196:osd.6> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.6.service 2024-04-05T02:47:32.929 INFO:tasks.cephadm:Deploying osd.7 on smithi196 with /dev/vg_nvme/lv_1... 2024-04-05T02:47:32.929 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-05T02:47:33.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[29558]: from='osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T02:47:33.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[29558]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T02:47:33.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[41818]: from='osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[41818]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:32 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:33.231 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:47:34.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-mon[33169]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:34.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-mon[33169]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-05T02:47:34.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-mon[33169]: osdmap e45: 7 total, 6 up, 7 in 2024-04-05T02:47:34.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:34.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-mon[33169]: from='osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:34.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-mon[33169]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:34.137 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T02:47:33.932+0000 7f4fec55c700 -1 osd.6 0 waiting for initial osdmap 2024-04-05T02:47:34.137 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 02:47:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T02:47:33.938+0000 7f4fe6b56700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:47:34.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[29558]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:34.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[29558]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[29558]: osdmap e45: 7 total, 6 up, 7 in 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[29558]: from='osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[29558]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[41818]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[41818]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[41818]: osdmap e45: 7 total, 6 up, 7 in 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[41818]: from='osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:33 smithi150 ceph-mon[41818]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[29558]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:47:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[29558]: osdmap e46: 7 total, 6 up, 7 in 2024-04-05T02:47:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[41818]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:47:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[41818]: osdmap e46: 7 total, 6 up, 7 in 2024-04-05T02:47:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:34 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:35.371 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:34 smithi196 ceph-mon[33169]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:47:35.372 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:34 smithi196 ceph-mon[33169]: osdmap e46: 7 total, 6 up, 7 in 2024-04-05T02:47:35.372 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:35.372 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:34 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:36.103 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:35 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:47:36.103 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:35 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:47:36.103 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:35 smithi196 ceph-mon[33169]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:36.103 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:35 smithi196 ceph-mon[33169]: osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024] boot 2024-04-05T02:47:36.104 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:35 smithi196 ceph-mon[33169]: osdmap e47: 7 total, 7 up, 7 in 2024-04-05T02:47:36.104 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:35 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[29558]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[29558]: osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024] boot 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[29558]: osdmap e47: 7 total, 7 up, 7 in 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[41818]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[41818]: osd.6 [v2:172.21.15.196:6816/471833024,v1:172.21.15.196:6817/471833024] boot 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[41818]: osdmap e47: 7 total, 7 up, 7 in 2024-04-05T02:47:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:35 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:47:37.085 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-05T02:47:37.085 INFO:teuthology.orchestra.run.smithi196.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-05T02:47:37.085 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10+0 records in 2024-04-05T02:47:37.085 INFO:teuthology.orchestra.run.smithi196.stderr:10+0 records out 2024-04-05T02:47:37.085 INFO:teuthology.orchestra.run.smithi196.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0149135 s, 703 MB/s 2024-04-05T02:47:37.085 INFO:teuthology.orchestra.run.smithi196.stderr:--> Zapping successful for: 2024-04-05T02:47:37.085 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:36 smithi196 ceph-mon[33169]: osdmap e48: 7 total, 7 up, 7 in 2024-04-05T02:47:37.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:36 smithi150 ceph-mon[29558]: osdmap e48: 7 total, 7 up, 7 in 2024-04-05T02:47:37.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:36 smithi150 ceph-mon[41818]: osdmap e48: 7 total, 7 up, 7 in 2024-04-05T02:47:38.060 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:37 smithi196 ceph-mon[33169]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:38.060 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:37 smithi196 ceph-mon[33169]: osdmap e49: 7 total, 7 up, 7 in 2024-04-05T02:47:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:37 smithi150 ceph-mon[29558]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:37 smithi150 ceph-mon[29558]: osdmap e49: 7 total, 7 up, 7 in 2024-04-05T02:47:38.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:37 smithi150 ceph-mon[41818]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:37 smithi150 ceph-mon[41818]: osdmap e49: 7 total, 7 up, 7 in 2024-04-05T02:47:38.588 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch daemon add osd smithi196:vg_nvme/lv_1 2024-04-05T02:47:38.836 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: osdmap e50: 7 total, 7 up, 7 in 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi196 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi196 to 5696M 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:39.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[29558]: osdmap e51: 7 total, 7 up, 7 in 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: osdmap e50: 7 total, 7 up, 7 in 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi196 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi196 to 5696M 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:38 smithi150 ceph-mon[41818]: osdmap e51: 7 total, 7 up, 7 in 2024-04-05T02:47:39.232 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: osdmap e50: 7 total, 7 up, 7 in 2024-04-05T02:47:39.232 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi196 2024-04-05T02:47:39.232 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi196 to 5696M 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:39.233 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:38 smithi196 ceph-mon[33169]: osdmap e51: 7 total, 7 up, 7 in 2024-04-05T02:47:40.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:39 smithi150 ceph-mon[29558]: pgmap v156: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-05T02:47:40.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:39 smithi150 ceph-mon[29558]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-05T02:47:40.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:39 smithi150 ceph-mon[41818]: pgmap v156: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-05T02:47:40.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:39 smithi150 ceph-mon[41818]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-05T02:47:40.264 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:39 smithi196 ceph-mon[33169]: pgmap v156: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-05T02:47:40.264 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:39 smithi196 ceph-mon[33169]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-04-05T02:47:41.221 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:47:41.221 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:47:41.221 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:40 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:41.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:47:41.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:47:41.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:40 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:41.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-05T02:47:41.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-05T02:47:41.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:40 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:42.128 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:41 smithi196 ceph-mon[33169]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:47:42.128 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:41 smithi196 ceph-mon[33169]: pgmap v158: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-05T02:47:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:41 smithi150 ceph-mon[29558]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:47:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:41 smithi150 ceph-mon[29558]: pgmap v158: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-05T02:47:42.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:41 smithi150 ceph-mon[41818]: from='client.24289 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi196:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:47:42.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:41 smithi150 ceph-mon[41818]: pgmap v158: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-04-05T02:47:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:43 smithi196 ceph-mon[33169]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-05T02:47:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:43 smithi196 ceph-mon[33169]: Cluster is now healthy 2024-04-05T02:47:43.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:43 smithi150 ceph-mon[29558]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-05T02:47:43.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:43 smithi150 ceph-mon[29558]: Cluster is now healthy 2024-04-05T02:47:43.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:43 smithi150 ceph-mon[41818]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-04-05T02:47:43.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:43 smithi150 ceph-mon[41818]: Cluster is now healthy 2024-04-05T02:47:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T02:47:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/1359542312' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]: dispatch 2024-04-05T02:47:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]: dispatch 2024-04-05T02:47:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]': finished 2024-04-05T02:47:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: osdmap e52: 8 total, 7 up, 8 in 2024-04-05T02:47:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:47:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:44 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/149051787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/1359542312' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]: dispatch 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]: dispatch 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]': finished 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: osdmap e52: 8 total, 7 up, 8 in 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/149051787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:47:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-05T02:47:44.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/1359542312' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]: dispatch 2024-04-05T02:47:44.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]: dispatch 2024-04-05T02:47:44.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5267c69d-e13b-45b4-a15e-91a620aa0647"}]': finished 2024-04-05T02:47:44.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: osdmap e52: 8 total, 7 up, 8 in 2024-04-05T02:47:44.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:47:44.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:44 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/149051787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-05T02:47:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:46 smithi196 ceph-mon[33169]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-05T02:47:46.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:46 smithi150 ceph-mon[29558]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-05T02:47:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:46 smithi150 ceph-mon[41818]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-05T02:47:48.116 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:48 smithi196 ceph-mon[33169]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:47:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:48 smithi150 ceph-mon[29558]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:47:48.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:48 smithi150 ceph-mon[41818]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:47:50.176 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:50 smithi196 ceph-mon[33169]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T02:47:50.176 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T02:47:50.176 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:50 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:50 smithi150 ceph-mon[41818]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T02:47:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T02:47:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:50 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:50 smithi150 ceph-mon[29558]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T02:47:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T02:47:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:50 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:51.241 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:51 smithi196 ceph-mon[33169]: Deploying daemon osd.7 on smithi196 2024-04-05T02:47:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:51 smithi150 ceph-mon[29558]: Deploying daemon osd.7 on smithi196 2024-04-05T02:47:51.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:51 smithi150 ceph-mon[41818]: Deploying daemon osd.7 on smithi196 2024-04-05T02:47:52.252 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:52 smithi196 ceph-mon[33169]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T02:47:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:52 smithi150 ceph-mon[29558]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T02:47:52.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:52 smithi150 ceph-mon[41818]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-05T02:47:54.220 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:54 smithi196 ceph-mon[33169]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:54.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:54 smithi150 ceph-mon[29558]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:54 smithi150 ceph-mon[41818]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:55 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:55 smithi196 ceph-mon[33169]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[29558]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:55 smithi150 ceph-mon[41818]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[29558]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:58.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:58.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T02:47:58.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:57 smithi150 ceph-mon[41818]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T02:47:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:57 smithi196 ceph-mon[33169]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-05T02:47:58.507 INFO:teuthology.orchestra.run.smithi196.stdout:Created osd(s) 7 on host 'smithi196' 2024-04-05T02:47:59.117 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: osdmap e53: 8 total, 7 up, 8 in 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:59.118 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:58 smithi196 ceph-mon[33169]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:59.140 DEBUG:teuthology.orchestra.run.smithi196:osd.7> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.7.service 2024-04-05T02:47:59.143 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-05T02:47:59.143 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd stat -f json 2024-04-05T02:47:59.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:59.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: osdmap e53: 8 total, 7 up, 8 in 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[41818]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-05T02:47:59.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: osdmap e53: 8 total, 7 up, 8 in 2024-04-05T02:47:59.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:47:59.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:59.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:58 smithi150 ceph-mon[29558]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]: dispatch 2024-04-05T02:47:59.407 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:00.040 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-mon[33169]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:48:00.041 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-mon[33169]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:48:00.041 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-mon[33169]: osdmap e54: 8 total, 7 up, 8 in 2024-04-05T02:48:00.041 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:00.041 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:00.041 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T02:47:59.723+0000 7f0cfa3a8700 -1 osd.7 0 waiting for initial osdmap 2024-04-05T02:48:00.041 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 02:47:59 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T02:47:59.731+0000 7f0cf41a1700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T02:48:00.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[41818]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:48:00.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[41818]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:48:00.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[41818]: osdmap e54: 8 total, 7 up, 8 in 2024-04-05T02:48:00.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:00.161 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:00.161 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[29558]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-04-05T02:48:00.161 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[29558]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi196", "root=default"]}]': finished 2024-04-05T02:48:00.161 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[29558]: osdmap e54: 8 total, 7 up, 8 in 2024-04-05T02:48:00.161 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:00.161 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:47:59 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:01.268 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: purged_snaps scrub starts 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: purged_snaps scrub ok 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114] boot 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: osdmap e55: 8 total, 8 up, 8 in 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4093549591' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-05T02:48:01.852 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: purged_snaps scrub starts 2024-04-05T02:48:01.853 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: purged_snaps scrub ok 2024-04-05T02:48:01.853 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114] boot 2024-04-05T02:48:01.853 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: osdmap e55: 8 total, 8 up, 8 in 2024-04-05T02:48:01.853 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:01.853 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:01.853 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:01 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4093549591' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-05T02:48:01.877 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1712285280,"num_in_osds":8,"osd_in_since":1712285263,"num_remapped_pgs":1} 2024-04-05T02:48:01.877 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: purged_snaps scrub starts 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: purged_snaps scrub ok 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: osd.7 [v2:172.21.15.196:6824/3556112114,v1:172.21.15.196:6825/3556112114] boot 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: osdmap e55: 8 total, 8 up, 8 in 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:02.106 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:01 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4093549591' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-05T02:48:02.130 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: osdmap e56: 8 total, 8 up, 8 in 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: Detected new or changed devices on smithi196 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi196 to 4272M 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: osdmap e56: 8 total, 8 up, 8 in 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: Detected new or changed devices on smithi196 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi196 to 4272M 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:02.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:02.980 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:02 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: osdmap e56: 8 total, 8 up, 8 in 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: Detected new or changed devices on smithi196 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:03.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi196 to 4272M 2024-04-05T02:48:03.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:03.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:03.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:03.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:02 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:03.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:03 smithi150 ceph-mon[29558]: osdmap e57: 8 total, 8 up, 8 in 2024-04-05T02:48:03.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:03 smithi150 ceph-mon[29558]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:03.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:03 smithi150 ceph-mon[29558]: osdmap e58: 8 total, 8 up, 8 in 2024-04-05T02:48:03.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:03 smithi150 ceph-mon[41818]: osdmap e57: 8 total, 8 up, 8 in 2024-04-05T02:48:03.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:03 smithi150 ceph-mon[41818]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:03.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:03 smithi150 ceph-mon[41818]: osdmap e58: 8 total, 8 up, 8 in 2024-04-05T02:48:04.075 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:48:04.076 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":58,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T02:48:03.199324+0000","last_up_change":"2024-04-05T02:48:00.713775+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6803","nonce":4116662696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6805","nonce":4116662696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6809","nonce":4116662696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6807","nonce":4116662696}]},"public_addr":"172.21.15.150:6803/4116662696","cluster_addr":"172.21.15.150:6805/4116662696","heartbeat_back_addr":"172.21.15.150:6809/4116662696","heartbeat_front_addr":"172.21.15.150:6807/4116662696","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6811","nonce":3820311304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6813","nonce":3820311304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6817","nonce":3820311304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6815","nonce":3820311304}]},"public_addr":"172.21.15.150:6811/3820311304","cluster_addr":"172.21.15.150:6813/3820311304","heartbeat_back_addr":"172.21.15.150:6817/3820311304","heartbeat_front_addr":"172.21.15.150:6815/3820311304","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6819","nonce":3881763618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6821","nonce":3881763618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6825","nonce":3881763618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6823","nonce":3881763618}]},"public_addr":"172.21.15.150:6819/3881763618","cluster_addr":"172.21.15.150:6821/3881763618","heartbeat_back_addr":"172.21.15.150:6825/3881763618","heartbeat_front_addr":"172.21.15.150:6823/3881763618","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6827","nonce":2956472788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6829","nonce":2956472788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6833","nonce":2956472788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6831","nonce":2956472788}]},"public_addr":"172.21.15.150:6827/2956472788","cluster_addr":"172.21.15.150:6829/2956472788","heartbeat_back_addr":"172.21.15.150:6833/2956472788","heartbeat_front_addr":"172.21.15.150:6831/2956472788","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T02:48:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:03 smithi196 ceph-mon[33169]: osdmap e57: 8 total, 8 up, 8 in 2024-04-05T02:48:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:03 smithi196 ceph-mon[33169]: pgmap v175: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:03 smithi196 ceph-mon[33169]: osdmap e58: 8 total, 8 up, 8 in 2024-04-05T02:48:04.657 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-05T02:45:59.176818+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-05T02:48:04.657 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-05T02:48:04.909 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:04 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1195472882' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:04 smithi150 ceph-mon[29558]: osdmap e59: 8 total, 8 up, 8 in 2024-04-05T02:48:04.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:04 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1195472882' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:04.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:04 smithi150 ceph-mon[41818]: osdmap e59: 8 total, 8 up, 8 in 2024-04-05T02:48:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:04 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1195472882' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:05.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:04 smithi196 ceph-mon[33169]: osdmap e59: 8 total, 8 up, 8 in 2024-04-05T02:48:06.048 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:05 smithi150 ceph-mon[29558]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:06.048 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:05 smithi150 ceph-mon[41818]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:06.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:05 smithi196 ceph-mon[33169]: pgmap v178: 1 pgs: 1 remapped+peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:06.747 INFO:teuthology.orchestra.run.smithi150.stdout:pg_num: 1 2024-04-05T02:48:07.055 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:06 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1262546167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-05T02:48:07.055 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:06 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1262546167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-05T02:48:07.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:06 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1262546167' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-05T02:48:07.438 INFO:tasks.cephadm:Adding prometheus.a on smithi196 2024-04-05T02:48:07.439 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch apply prometheus '1;smithi196=a' 2024-04-05T02:48:07.701 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:48:07.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:07 smithi150 ceph-mon[41818]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-05T02:48:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:07 smithi150 ceph-mon[29558]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-05T02:48:08.109 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:07 smithi196 ceph-mon[33169]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-04-05T02:48:09.540 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled prometheus update... 2024-04-05T02:48:10.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:09 smithi196 ceph-mon[33169]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-05T02:48:10.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:10.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:10.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:10.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:10.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:09 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:10.187 DEBUG:teuthology.orchestra.run.smithi196:prometheus.a> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service 2024-04-05T02:48:10.190 INFO:tasks.cephadm:Adding node-exporter.a on smithi150 2024-04-05T02:48:10.190 INFO:tasks.cephadm:Adding node-exporter.b on smithi196 2024-04-05T02:48:10.190 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch apply node-exporter '2;smithi150=a;smithi196=b' 2024-04-05T02:48:10.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[41818]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-05T02:48:10.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:10.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:10.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[29558]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 KiB/s, 0 objects/s recovering 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:10.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:09 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:10.489 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:48:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:10 smithi196 ceph-mon[33169]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi196=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:11.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:10 smithi196 ceph-mon[33169]: Saving service prometheus spec with placement smithi196=a;count:1 2024-04-05T02:48:11.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:10 smithi196 ceph-mon[33169]: Deploying daemon prometheus.a on smithi196 2024-04-05T02:48:11.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:10 smithi150 ceph-mon[29558]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi196=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:11.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:10 smithi150 ceph-mon[29558]: Saving service prometheus spec with placement smithi196=a;count:1 2024-04-05T02:48:11.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:10 smithi150 ceph-mon[29558]: Deploying daemon prometheus.a on smithi196 2024-04-05T02:48:11.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:10 smithi150 ceph-mon[41818]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi196=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:11.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:10 smithi150 ceph-mon[41818]: Saving service prometheus spec with placement smithi196=a;count:1 2024-04-05T02:48:11.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:10 smithi150 ceph-mon[41818]: Deploying daemon prometheus.a on smithi196 2024-04-05T02:48:12.079 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:11 smithi196 ceph-mon[33169]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:48:12.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:11 smithi150 ceph-mon[29558]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:48:12.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:11 smithi150 ceph-mon[41818]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-05T02:48:12.737 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled node-exporter update... 2024-04-05T02:48:13.734 DEBUG:teuthology.orchestra.run.smithi150:node-exporter.a> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service 2024-04-05T02:48:13.737 DEBUG:teuthology.orchestra.run.smithi196:node-exporter.b> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service 2024-04-05T02:48:13.739 INFO:tasks.cephadm:Adding alertmanager.a on smithi150 2024-04-05T02:48:13.740 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch apply alertmanager '1;smithi150=a' 2024-04-05T02:48:14.061 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:48:14.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:13 smithi196 ceph-mon[33169]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi150=a;smithi196=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:14.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:13 smithi196 ceph-mon[33169]: Saving service node-exporter spec with placement smithi150=a;smithi196=b;count:2 2024-04-05T02:48:14.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:13 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:14.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:13 smithi196 ceph-mon[33169]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-05T02:48:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[29558]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi150=a;smithi196=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[29558]: Saving service node-exporter spec with placement smithi150=a;smithi196=b;count:2 2024-04-05T02:48:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[29558]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-05T02:48:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[41818]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi150=a;smithi196=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[41818]: Saving service node-exporter spec with placement smithi150=a;smithi196=b;count:2 2024-04-05T02:48:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:13 smithi150 ceph-mon[41818]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2024-04-05T02:48:16.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:15 smithi196 ceph-mon[33169]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-05T02:48:16.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:15 smithi150 ceph-mon[29558]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-05T02:48:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:15 smithi150 ceph-mon[41818]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 KiB/s, 0 objects/s recovering 2024-04-05T02:48:18.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:17 smithi196 ceph-mon[33169]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-05T02:48:18.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:17 smithi150 ceph-mon[29558]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-05T02:48:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:17 smithi150 ceph-mon[41818]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-05T02:48:19.154 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled alertmanager update... 2024-04-05T02:48:20.165 DEBUG:teuthology.orchestra.run.smithi150:alertmanager.a> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@alertmanager.a.service 2024-04-05T02:48:20.168 INFO:tasks.cephadm:Adding grafana.a on smithi196 2024-04-05T02:48:20.168 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph orch apply grafana '1;smithi196=a' 2024-04-05T02:48:20.408 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:20 smithi196 ceph-mon[33169]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:20.408 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:20 smithi196 ceph-mon[33169]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:20.409 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:20 smithi196 ceph-mon[33169]: Saving service alertmanager spec with placement smithi150=a;count:1 2024-04-05T02:48:20.409 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:20 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:20.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[29558]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[29558]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[29558]: Saving service alertmanager spec with placement smithi150=a;count:1 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[41818]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[41818]: from='client.24337 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi150=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[41818]: Saving service alertmanager spec with placement smithi150=a;count:1 2024-04-05T02:48:20.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:20 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:22.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:22 smithi196 ceph-mon[33169]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:22.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:22 smithi150 ceph-mon[29558]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:22.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:22 smithi150 ceph-mon[41818]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:22.729 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:48:24.392 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-mon[33169]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:24.392 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 podman[62087]: 2024-04-05 02:48:24.269178859 +0000 UTC m=+0.098463986 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T02:48:24.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:24 smithi150 ceph-mon[41818]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:24.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:24 smithi150 ceph-mon[29558]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:24.573 INFO:teuthology.orchestra.run.smithi196.stdout:Scheduled grafana update... 2024-04-05T02:48:24.886 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 podman[62087]: 2024-04-05 02:48:24.526454702 +0000 UTC m=+0.355739817 container create 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:48:25.232 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 podman[62087]: 2024-04-05 02:48:24.959609012 +0000 UTC m=+0.788894138 container init 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:48:25.232 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 podman[62087]: 2024-04-05 02:48:24.965335092 +0000 UTC m=+0.794620207 container start 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:48:25.232 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.989Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.989Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.989Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi196 (none))" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.989Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.989Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.990Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.991Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.992Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.992Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.994Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.994Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.424µs 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.994Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.995Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.995Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.629µs wal_replay_duration=1.244976ms wbl_replay_duration=117ns total_replay_duration=1.280048ms 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.996Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T02:48:25.233 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.996Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T02:48:25.234 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:24.996Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T02:48:25.234 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:25.011Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.804185ms db_storage=744ns remote_storage=1.027µs web_handler=262ns query_engine=528ns scrape=409.242µs scrape_sd=16.086µs notify=666ns notify_sd=1.377µs rules=14.223205ms tracing=7.178µs 2024-04-05T02:48:25.234 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:25.011Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T02:48:25.234 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:48:25.011Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T02:48:25.234 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:25 smithi196 bash[62087]: 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 2024-04-05T02:48:25.234 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:48:25 smithi196 systemd[1]: Started Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:48:25.522 DEBUG:teuthology.orchestra.run.smithi196:grafana.a> sudo journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service 2024-04-05T02:48:25.524 INFO:tasks.cephadm:Setting up client nodes... 2024-04-05T02:48:25.525 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-05T02:48:25.787 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi196=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: Saving service grafana spec with placement smithi196=a;count:1 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:25 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-05T02:48:25.909 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi196=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:25.909 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: Saving service grafana spec with placement smithi196=a;count:1 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi196=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: Saving service grafana spec with placement smithi196=a;count:1 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:25.910 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.911 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.911 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' 2024-04-05T02:48:25.911 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:25 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-05T02:48:26.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setuser ceph since I am not root 2024-04-05T02:48:26.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setgroup ceph since I am not root 2024-04-05T02:48:26.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:26 smithi150 ceph-mgr[29835]: -- 172.21.15.150:0/3493934409 <== mon.0 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56280e59c9c0 con 0x56280e572c00 2024-04-05T02:48:26.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ignoring --setuser ceph since I am not root 2024-04-05T02:48:26.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ignoring --setgroup ceph since I am not root 2024-04-05T02:48:26.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:26 smithi196 ceph-mgr[35298]: -- 172.21.15.196:0/1891355588 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5619101349c0 con 0x56191010a800 2024-04-05T02:48:26.908 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:26.669+0000 7f3bca7b01c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:48:26.909 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:26.763+0000 7f3bca7b01c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:48:26.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:26.684+0000 7f33ffeed1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:48:26.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:26.775+0000 7f33ffeed1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:48:27.242 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-mon[41818]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-05T02:48:27.242 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-mon[41818]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-05T02:48:27.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-mon[29558]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-05T02:48:27.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-mon[29558]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-05T02:48:27.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-mon[33169]: from='mgr.14152 172.21.15.150:0/3880496303' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-05T02:48:27.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-mon[33169]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-05T02:48:27.500 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:27.419+0000 7f33ffeed1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:48:27.750 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:27.418+0000 7f3bca7b01c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:48:27.750 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:27.509+0000 7f3bca7b01c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:48:27.750 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:27.595+0000 7f3bca7b01c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:48:27.750 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:27.749+0000 7f3bca7b01c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:48:27.786 INFO:teuthology.orchestra.run.smithi150.stdout:[client.0] 2024-04-05T02:48:27.786 INFO:teuthology.orchestra.run.smithi150.stdout: key = AQB7Zg9m8UCrLhAAkS/gOroNJ4v51ebO5smBhQ== 2024-04-05T02:48:27.787 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:27.514+0000 7f33ffeed1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:48:27.787 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:27.599+0000 7f33ffeed1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:48:27.787 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:27.749+0000 7f33ffeed1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:48:28.004 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:27.831+0000 7f3bca7b01c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:48:28.004 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:28.003+0000 7f3bca7b01c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:48:28.137 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:27 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:27.836+0000 7f33ffeed1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:48:28.137 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:28.007+0000 7f33ffeed1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:48:28.267 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1468710030' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:28.267 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1468710030' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T02:48:28.267 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:28.183+0000 7f3bca7b01c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:48:28.456 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1468710030' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:28.457 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1468710030' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T02:48:28.457 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1468710030' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:28.457 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1468710030' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T02:48:28.457 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:28.192+0000 7f33ffeed1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:48:28.457 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:28.276+0000 7f33ffeed1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:48:28.457 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:28.370+0000 7f33ffeed1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:48:28.530 DEBUG:teuthology.orchestra.run.smithi150:> set -ex 2024-04-05T02:48:28.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-05T02:48:28.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-05T02:48:28.576 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-05T02:48:28.601 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:28.266+0000 7f3bca7b01c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:48:28.601 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:28.358+0000 7f3bca7b01c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:48:28.601 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:28.442+0000 7f3bca7b01c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:48:28.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:28 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:28.456+0000 7f33ffeed1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:48:28.847 INFO:teuthology.orchestra.run.smithi196.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.b/config 2024-04-05T02:48:29.636 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:29.529+0000 7f3bca7b01c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:48:29.830 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:29.511+0000 7f33ffeed1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:48:29.830 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:29.640+0000 7f33ffeed1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:48:29.830 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:29.830+0000 7f33ffeed1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:48:30.054 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:29.659+0000 7f3bca7b01c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:48:30.055 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:29.852+0000 7f3bca7b01c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:48:30.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:30.058+0000 7f33ffeed1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:48:30.340 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:30.082+0000 7f3bca7b01c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:48:30.340 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:30.265+0000 7f3bca7b01c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:48:30.616 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:30.448+0000 7f3bca7b01c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:48:30.677 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:30.233+0000 7f33ffeed1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:48:30.677 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:30.414+0000 7f33ffeed1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:48:30.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:30.716+0000 7f3bca7b01c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:48:30.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:30.804+0000 7f3bca7b01c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:48:30.904 INFO:teuthology.orchestra.run.smithi196.stdout:[client.1] 2024-04-05T02:48:30.904 INFO:teuthology.orchestra.run.smithi196.stdout: key = AQB+Zg9mPHC8NRAAHyHAt0//oQFv+ZSO9cl97Q== 2024-04-05T02:48:30.952 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:30.676+0000 7f33ffeed1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:48:30.953 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:30.764+0000 7f33ffeed1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:48:31.205 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-mon[33169]: from='client.? 172.21.15.196:0/1197784455' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:31.205 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:31.205 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:30 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T02:48:31.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:31.154+0000 7f33ffeed1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:48:31.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-mon[41818]: from='client.? 172.21.15.196:0/1197784455' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:31.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:31.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T02:48:31.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-mon[29558]: from='client.? 172.21.15.196:0/1197784455' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:31.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-05T02:48:31.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:30 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-05T02:48:31.503 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:31.204+0000 7f3bca7b01c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:48:31.531 DEBUG:teuthology.orchestra.run.smithi196:> set -ex 2024-04-05T02:48:31.531 DEBUG:teuthology.orchestra.run.smithi196:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-05T02:48:31.531 DEBUG:teuthology.orchestra.run.smithi196:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-05T02:48:31.613 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-05T02:48:31.614 DEBUG:teuthology.orchestra.run.smithi150:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-05T02:48:31.916 INFO:teuthology.misc.health.smithi150.stdout: 2024-04-05T02:48:31.916 INFO:teuthology.misc.health.smithi150.stdout:{"epoch":59,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T02:48:04.200752+0000","last_up_change":"2024-04-05T02:48:00.713775+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6803","nonce":4116662696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6805","nonce":4116662696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6809","nonce":4116662696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6807","nonce":4116662696}]},"public_addr":"172.21.15.150:6803/4116662696","cluster_addr":"172.21.15.150:6805/4116662696","heartbeat_back_addr":"172.21.15.150:6809/4116662696","heartbeat_front_addr":"172.21.15.150:6807/4116662696","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6811","nonce":3820311304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6813","nonce":3820311304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6817","nonce":3820311304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6815","nonce":3820311304}]},"public_addr":"172.21.15.150:6811/3820311304","cluster_addr":"172.21.15.150:6813/3820311304","heartbeat_back_addr":"172.21.15.150:6817/3820311304","heartbeat_front_addr":"172.21.15.150:6815/3820311304","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6819","nonce":3881763618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6821","nonce":3881763618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6825","nonce":3881763618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6823","nonce":3881763618}]},"public_addr":"172.21.15.150:6819/3881763618","cluster_addr":"172.21.15.150:6821/3881763618","heartbeat_back_addr":"172.21.15.150:6825/3881763618","heartbeat_front_addr":"172.21.15.150:6823/3881763618","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6827","nonce":2956472788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6829","nonce":2956472788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6833","nonce":2956472788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6831","nonce":2956472788}]},"public_addr":"172.21.15.150:6827/2956472788","cluster_addr":"172.21.15.150:6829/2956472788","heartbeat_back_addr":"172.21.15.150:6833/2956472788","heartbeat_front_addr":"172.21.15.150:6831/2956472788","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T02:48:31.928 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-05T02:48:31.928 INFO:tasks.cephadm:Creating RBD pool 2024-04-05T02:48:31.928 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-05T02:48:32.181 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:31 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4207841794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:32.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:31 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4207841794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:32.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:31.972+0000 7f33ffeed1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:48:32.284 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:31 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4207841794' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:32.284 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:31.992+0000 7f3bca7b01c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:48:32.284 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:32.283+0000 7f3bca7b01c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:48:32.566 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:32.286+0000 7f33ffeed1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:48:32.877 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:32.615+0000 7f33ffeed1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:48:32.877 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:32.812+0000 7f33ffeed1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:48:32.877 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:32] ENGINE Bus STARTING 2024-04-05T02:48:32.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:32.599+0000 7f3bca7b01c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:48:32.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:32.791+0000 7f3bca7b01c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:48:32.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: [05/Apr/2024:02:48:32] ENGINE Bus STARTING 2024-04-05T02:48:33.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: CherryPy Checker: 2024-04-05T02:48:33.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: The Application mounted at '' has an empty config. 2024-04-05T02:48:33.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:48:33.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:33] ENGINE Serving on http://:::9283 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:33] ENGINE Bus STARTED 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: Standby manager daemon x restarted 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: Standby manager daemon x started 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:48:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: Active manager daemon y restarted 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: Activating manager daemon y 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: osdmap e60: 8 total, 8 up, 8 in 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: mgrmap e17: y(active, starting, since 0.00590812s), standbys: x 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:48:33.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: Manager daemon y is now available 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:33.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: Standby manager daemon x restarted 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: Standby manager daemon x started 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: Active manager daemon y restarted 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: Activating manager daemon y 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: osdmap e60: 8 total, 8 up, 8 in 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: mgrmap e17: y(active, starting, since 0.00590812s), standbys: x 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:48:33.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:48:33.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: Manager daemon y is now available 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:48:33.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:33.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:48:33.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:48:33.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:48:33.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:48:33.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:32 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: CherryPy Checker: 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: The Application mounted at '' has an empty config. 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: [05/Apr/2024:02:48:32] ENGINE Serving on http://:::9283 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: [05/Apr/2024:02:48:32] ENGINE Bus STARTED 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: Standby manager daemon x restarted 2024-04-05T02:48:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: Standby manager daemon x started 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/2505854153' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: Active manager daemon y restarted 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: Activating manager daemon y 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: osdmap e60: 8 total, 8 up, 8 in 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: mgrmap e17: y(active, starting, since 0.00590812s), standbys: x 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:48:33.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: Manager daemon y is now available 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:48:33.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:48:33.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:48:33.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:48:33.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:32 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:48:34.692 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:34 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:34.692 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:34 smithi196 ceph-mon[33169]: mgrmap e18: y(active, since 1.01253s), standbys: x 2024-04-05T02:48:34.692 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:34 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2883685533' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T02:48:34.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:34 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:34.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:34 smithi150 ceph-mon[29558]: mgrmap e18: y(active, since 1.01253s), standbys: x 2024-04-05T02:48:34.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:34 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2883685533' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T02:48:34.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:34 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:34.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:34 smithi150 ceph-mon[41818]: mgrmap e18: y(active, since 1.01253s), standbys: x 2024-04-05T02:48:34.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:34 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2883685533' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T02:48:34.834 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'rbd' created 2024-04-05T02:48:35.501 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-05T02:48:35.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: [05/Apr/2024:02:48:33] ENGINE Bus STARTING 2024-04-05T02:48:35.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: [05/Apr/2024:02:48:33] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:48:35.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: [05/Apr/2024:02:48:33] ENGINE Bus STARTED 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: osdmap e61: 8 total, 8 up, 8 in 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: [05/Apr/2024:02:48:33] ENGINE Bus STARTING 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: [05/Apr/2024:02:48:33] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: [05/Apr/2024:02:48:33] ENGINE Bus STARTED 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: osdmap e61: 8 total, 8 up, 8 in 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:35 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.763 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: [05/Apr/2024:02:48:33] ENGINE Bus STARTING 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: [05/Apr/2024:02:48:33] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: [05/Apr/2024:02:48:33] ENGINE Bus STARTED 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: osdmap e61: 8 total, 8 up, 8 in 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:35.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:35 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-05T02:48:36.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: osdmap e62: 8 total, 8 up, 8 in 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi196 to 4016M 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: osdmap e62: 8 total, 8 up, 8 in 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi196 to 4016M 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:48:36.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:36 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: osdmap e62: 8 total, 8 up, 8 in 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi196 to 4016M 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:48:36.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:36 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: osdmap e63: 8 total, 8 up, 8 in 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Deploying daemon alertmanager.a on smithi150 2024-04-05T02:48:38.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:48:38.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:37 smithi196 ceph-mon[33169]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-05T02:48:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:48:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:48:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:48:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:48:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: osdmap e63: 8 total, 8 up, 8 in 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Deploying daemon alertmanager.a on smithi150 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[29558]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:48:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: osdmap e63: 8 total, 8 up, 8 in 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Deploying daemon alertmanager.a on smithi150 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:48:38.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:37 smithi150 ceph-mon[41818]: mgrmap e20: y(active, since 4s), standbys: x 2024-04-05T02:48:38.859 INFO:teuthology.orchestra.run.smithi150.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-05T02:48:38.924 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:38 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2991617005' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:48:38.925 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:38 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2991617005' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:48:39.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:38 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2991617005' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:48:40.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:39 smithi196 ceph-mon[33169]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:39 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2991617005' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:48:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:39 smithi196 ceph-mon[33169]: osdmap e64: 8 total, 8 up, 8 in 2024-04-05T02:48:40.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:39 smithi150 ceph-mon[29558]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:39 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2991617005' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:48:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:39 smithi150 ceph-mon[29558]: osdmap e64: 8 total, 8 up, 8 in 2024-04-05T02:48:40.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:39 smithi150 ceph-mon[41818]: pgmap v9: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:40.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:39 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2991617005' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:48:40.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:39 smithi150 ceph-mon[41818]: osdmap e64: 8 total, 8 up, 8 in 2024-04-05T02:48:40.736 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-05T02:48:40.736 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-05T02:48:40.736 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-05T02:48:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:40 smithi150 ceph-mon[29558]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:48:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:40 smithi150 ceph-mon[29558]: Cluster is now healthy 2024-04-05T02:48:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:40 smithi150 ceph-mon[41818]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:48:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:40 smithi150 ceph-mon[41818]: Cluster is now healthy 2024-04-05T02:48:41.357 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:41.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:40 smithi196 ceph-mon[33169]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:48:41.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:40 smithi196 ceph-mon[33169]: Cluster is now healthy 2024-04-05T02:48:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:41 smithi150 ceph-mon[29558]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-05T02:48:42.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:41 smithi150 ceph-mon[41818]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-05T02:48:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:41 smithi196 ceph-mon[33169]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-04-05T02:48:42.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:48:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:48:44.031 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:43 smithi150 ceph-mon[41818]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T02:48:44.031 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:43 smithi150 ceph-mon[29558]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T02:48:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:43 smithi196 ceph-mon[33169]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-05T02:48:44.918 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:48:46.193 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:45 smithi150 ceph-mon[41818]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-05T02:48:46.193 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:45 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/183959549' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T02:48:46.193 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:45 smithi150 ceph-mon[29558]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-05T02:48:46.193 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:45 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/183959549' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T02:48:46.220 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":20,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":1834837291},{"type":"v1","addr":"172.21.15.150:6801","nonce":1834837291}]},"active_addr":"172.21.15.150:6801/1834837291","active_change":"2024-04-05T02:48:32.814886+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24335,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.150:8443/","prometheus":"http://172.21.15.150:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":11242322}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":198010180}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2745855992}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2603883274}]}]} 2024-04-05T02:48:46.224 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-05T02:48:46.224 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-05T02:48:46.224 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T02:48:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:45 smithi196 ceph-mon[33169]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 14 op/s 2024-04-05T02:48:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:45 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/183959549' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T02:48:46.832 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:47 smithi150 ceph-mon[41818]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T02:48:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:47 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:48:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:47 smithi150 ceph-mon[29558]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T02:48:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:47 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:48:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:47 smithi196 ceph-mon[33169]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T02:48:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:47 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:48:48.425 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 podman[73008]: 2024-04-05 02:48:48.205609463 +0000 UTC m=+0.564950163 container create 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:48:48.792 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 podman[73008]: 2024-04-05 02:48:48.541757291 +0000 UTC m=+0.901097988 container init 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 podman[73008]: 2024-04-05 02:48:48.547618417 +0000 UTC m=+0.906959123 container start 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.559Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.559Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.560Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.150 port=9094 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.561Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.584Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.584Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.587Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:48.587Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 bash[73008]: 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e 2024-04-05T02:48:48.793 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:48 smithi150 systemd[1]: Started Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:48:49.352 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:48:49.352 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":64,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T02:48:38.841597+0000","last_up_change":"2024-04-05T02:48:00.713775+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T02:48:34.437290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6803","nonce":4116662696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6805","nonce":4116662696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6809","nonce":4116662696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6807","nonce":4116662696}]},"public_addr":"172.21.15.150:6803/4116662696","cluster_addr":"172.21.15.150:6805/4116662696","heartbeat_back_addr":"172.21.15.150:6809/4116662696","heartbeat_front_addr":"172.21.15.150:6807/4116662696","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6811","nonce":3820311304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6813","nonce":3820311304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6817","nonce":3820311304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6815","nonce":3820311304}]},"public_addr":"172.21.15.150:6811/3820311304","cluster_addr":"172.21.15.150:6813/3820311304","heartbeat_back_addr":"172.21.15.150:6817/3820311304","heartbeat_front_addr":"172.21.15.150:6815/3820311304","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6819","nonce":3881763618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6821","nonce":3881763618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6825","nonce":3881763618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6823","nonce":3881763618}]},"public_addr":"172.21.15.150:6819/3881763618","cluster_addr":"172.21.15.150:6821/3881763618","heartbeat_back_addr":"172.21.15.150:6825/3881763618","heartbeat_front_addr":"172.21.15.150:6823/3881763618","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6827","nonce":2956472788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6829","nonce":2956472788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6833","nonce":2956472788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6831","nonce":2956472788}]},"public_addr":"172.21.15.150:6827/2956472788","cluster_addr":"172.21.15.150:6829/2956472788","heartbeat_back_addr":"172.21.15.150:6833/2956472788","heartbeat_front_addr":"172.21.15.150:6831/2956472788","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:6801/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3200004174":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/2202195129":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3549400810":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2441406623":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T02:48:49.353 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:49] ENGINE Bus STOPPING 2024-04-05T02:48:49.353 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:48:49.353 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:49] ENGINE Bus STOPPED 2024-04-05T02:48:49.353 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:49] ENGINE Bus STARTING 2024-04-05T02:48:49.353 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:49] ENGINE Serving on http://:::9283 2024-04-05T02:48:49.353 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:48:49] ENGINE Bus STARTED 2024-04-05T02:48:49.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: Deploying daemon grafana.a on smithi196 2024-04-05T02:48:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2108197860' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:49.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: Deploying daemon grafana.a on smithi196 2024-04-05T02:48:49.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:49 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2108197860' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:50.065 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-05T02:48:50.065 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T02:48:50.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: Deploying daemon grafana.a on smithi196 2024-04-05T02:48:50.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:49 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2108197860' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:50.319 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:50.694 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:50.562Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000706402s 2024-04-05T02:48:52.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:51 smithi196 ceph-mon[33169]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:52.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:51 smithi150 ceph-mon[29558]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:52.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:51 smithi150 ceph-mon[41818]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:52.257 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:48:52.257 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":64,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T02:48:38.841597+0000","last_up_change":"2024-04-05T02:48:00.713775+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T02:48:34.437290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6803","nonce":4116662696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6805","nonce":4116662696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6809","nonce":4116662696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6807","nonce":4116662696}]},"public_addr":"172.21.15.150:6803/4116662696","cluster_addr":"172.21.15.150:6805/4116662696","heartbeat_back_addr":"172.21.15.150:6809/4116662696","heartbeat_front_addr":"172.21.15.150:6807/4116662696","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6811","nonce":3820311304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6813","nonce":3820311304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6817","nonce":3820311304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6815","nonce":3820311304}]},"public_addr":"172.21.15.150:6811/3820311304","cluster_addr":"172.21.15.150:6813/3820311304","heartbeat_back_addr":"172.21.15.150:6817/3820311304","heartbeat_front_addr":"172.21.15.150:6815/3820311304","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6819","nonce":3881763618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6821","nonce":3881763618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6825","nonce":3881763618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6823","nonce":3881763618}]},"public_addr":"172.21.15.150:6819/3881763618","cluster_addr":"172.21.15.150:6821/3881763618","heartbeat_back_addr":"172.21.15.150:6825/3881763618","heartbeat_front_addr":"172.21.15.150:6823/3881763618","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6827","nonce":2956472788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6829","nonce":2956472788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6833","nonce":2956472788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6831","nonce":2956472788}]},"public_addr":"172.21.15.150:6827/2956472788","cluster_addr":"172.21.15.150:6829/2956472788","heartbeat_back_addr":"172.21.15.150:6833/2956472788","heartbeat_front_addr":"172.21.15.150:6831/2956472788","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:6801/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3200004174":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/2202195129":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3549400810":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2441406623":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T02:48:52.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:48:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:48:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:48:52.949 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-05T02:48:52.950 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-05T02:48:52.951 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-05T02:48:52.952 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-05T02:48:52.952 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-05T02:48:52.953 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-05T02:48:52.954 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-05T02:48:52.954 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-05T02:48:53.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:52 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2139711259' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:53.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:52 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:53.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:52 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2139711259' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:53.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:52 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:53.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:52 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2139711259' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:48:53.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:52 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:48:53.489 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.492 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.493 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.495 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.495 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.498 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.502 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:53.503 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:48:54.198 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:53 smithi150 ceph-mon[29558]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:54.199 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:53 smithi150 ceph-mon[41818]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:54.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:53 smithi196 ceph-mon[33169]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:55.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:55 smithi150 ceph-mon[41818]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:55.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:55 smithi150 ceph-mon[29558]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:56.188 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:55 smithi196 ceph-mon[33169]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:57 smithi150 ceph-mon[29558]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:57.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:57 smithi150 ceph-mon[41818]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:57 smithi196 ceph-mon[33169]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:48:58.867 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:48:58 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:48:58.565Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003623667s 2024-04-05T02:49:00.173 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:48:59 smithi150 ceph-mon[41818]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:00.173 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:48:59 smithi150 ceph-mon[29558]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:48:59 smithi196 ceph-mon[33169]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:01.674 INFO:teuthology.orchestra.run.smithi150.stdout:167503724567 2024-04-05T02:49:01.674 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-05T02:49:01.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:01 smithi150 ceph-mon[29558]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:01 smithi196 ceph-mon[33169]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:02.165 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:01 smithi150 ceph-mon[41818]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:02.750 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:49:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:49:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:49:02.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:02 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:02 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:03.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:02 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:03.492 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 podman[64324]: 2024-04-05 02:49:03.373331436 +0000 UTC m=+0.134941763 container create 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , io.openshift.expose-services=, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vendor=Red Hat, Inc., description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, distribution-scope=public, name=ubi8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664) 2024-04-05T02:49:03.851 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 podman[64324]: 2024-04-05 02:49:03.600633977 +0000 UTC m=+0.362244258 container init 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, description=Ceph Grafana Container, vendor=Red Hat, Inc., vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.expose-services=, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.component=ubi8-container) 2024-04-05T02:49:03.851 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 podman[64324]: 2024-04-05 02:49:03.605686258 +0000 UTC m=+0.367296539 container start 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vcs-type=git, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664) 2024-04-05T02:49:03.851 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 bash[64324]: 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe 2024-04-05T02:49:03.852 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:03.852 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.757941895Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-05T02:49:03.852 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758129745Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-05T02:49:03.852 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758150595Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-05T02:49:03.852 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758157504Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-05T02:49:03.852 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758163422Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758169263Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758174921Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758180657Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.75818788Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758193655Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758199173Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.75820457Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.758210201Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=settings t=2024-04-05T02:49:03.75821585Z level=info msg="App mode production" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore t=2024-04-05T02:49:03.758261655Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore t=2024-04-05T02:49:03.75827749Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.759153714Z level=info msg="Starting DB migrations" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.759233227Z level=info msg="Executing migration" id="create migration_log table" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.759748364Z level=info msg="Executing migration" id="create user table" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.760183345Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-05T02:49:03.853 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.760633243Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.761051848Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.76154831Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.7619176Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.762657433Z level=info msg="Executing migration" id="create user table v2" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.763100284Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.763511891Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.763929897Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.764284975Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.764645753Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.765037688Z level=info msg="Executing migration" id="Update user table charset" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.765364048Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.765766966Z level=info msg="Executing migration" id="Add missing user data" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.76611512Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.766536359Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-05T02:49:03.854 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.766922043Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.76736159Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.768506046Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.768956028Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.769437452Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.769845359Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.770258887Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.770659327Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.77092516Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.771348433Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.771740877Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.772108728Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.772484629Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.773095531Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-05T02:49:03.855 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.773487172Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.773851171Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.774245025Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.774628413Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.775009791Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.777318623Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.777759468Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.778106504Z level=info msg="Executing migration" id="create star table" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.778485863Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.778913453Z level=info msg="Executing migration" id="create org table v1" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.779324293Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.779749047Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-05T02:49:03.856 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.780147154Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.78055821Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.78103802Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.781445949Z level=info msg="Executing migration" id="Update org table charset" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.781706424Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.781966113Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.782279737Z level=info msg="Executing migration" id="create dashboard table" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.782633837Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.783014558Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.783403714Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-05T02:49:03.857 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.783780536Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.78637949Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.788961406Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.79156151Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.791973742Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.792817097Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.793662938Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.794020358Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.794451546Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.79473441Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.795240868Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.795738038Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.79619725Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.796625623Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-05T02:49:03.858 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.79709583Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.797529989Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.797919093Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.79819148Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.798482994Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.798990435Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.799860693Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.800354562Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.800847792Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.801153927Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.801592287Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.801974111Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.802291333Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.802872371Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.803700111Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.804600837Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-05T02:49:03.859 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.805042965Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.805888066Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.806692692Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.807655533Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.808380008Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.809342333Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.8100915Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.81099626Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.811805089Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.812790288Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.813506531Z level=info msg="Executing migration" id="create data_source table" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.814410939Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.815103962Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.815525911Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.816408806Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.816927712Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-05T02:49:03.860 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.819453165Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.822182782Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.824988398Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.827620116Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.830309822Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.83295176Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.835873449Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.83864986Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.841162981Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.844044146Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.846726856Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.849613994Z level=info msg="Executing migration" id="Add uid column" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.85219507Z level=info msg="Executing migration" id="Update uid value" 2024-04-05T02:49:03.861 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.854962501Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-05T02:49:04.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-mon[33169]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:04.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.102 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.857589588Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.86042236Z level=info msg="Executing migration" id="create api_key table" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.863354672Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.865923456Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.868542717Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.871257763Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.873875594Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.876644714Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.879327531Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.882026735Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.882481064Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-05T02:49:04.103 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.88494021Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.885447605Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.887810442Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.888203656Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.890532978Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.89298006Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.89551677Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.898229145Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.898723296Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.901514004Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.902331314Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.903121491Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.904119824Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.90480573Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.905631152Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.907917196Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-05T02:49:04.104 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.910576772Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.913268599Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.916052085Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.919016928Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.91970871Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.922478953Z level=info msg="Executing migration" id="create quota table v1" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.922926296Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.925459889Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.92807214Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.928508995Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.931306036Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.934078081Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.936879248Z level=info msg="Executing migration" id="create session table" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.937399074Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.940281535Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.942995675Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-05T02:49:04.105 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.945709811Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.946153086Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.948694796Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.951557047Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.954262466Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.954744446Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.9575445Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.960256536Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.96290255Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.963236632Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.966144843Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.968755657Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.971542839Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-05T02:49:04.106 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.972032903Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-05T02:49:04.107 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.974721218Z level=info msg="Executing migration" id="create alert table v1" 2024-04-05T02:49:04.107 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.977553303Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.978208459Z level=info msg="Executing migration" id="add index alert state" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.978641465Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.979548227Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.980180908Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.980956182Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.981376286Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.982799008Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.983634293Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.984422331Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.984749064Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.985678838Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.986276347Z level=info msg="Executing migration" id="Add column is_default" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.986892437Z level=info msg="Executing migration" id="Add column frequency" 2024-04-05T02:49:04.108 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.987712337Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.988343383Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.989176648Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.989981489Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.990928541Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.991302546Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.992354562Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.993060343Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.99397133Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.994463622Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.995261811Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.995985463Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.996691076Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.997536155Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.998010724Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.998850398Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:03 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:03.999669752Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-05T02:49:04.109 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.000602124Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.001140597Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.002004421Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.002864268Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.003668554Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.00414646Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.00705359Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.00969969Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.012403483Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.013019703Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.015777197Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.018648794Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.021291258Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.022022276Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-05T02:49:04.110 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.024984513Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.027768941Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.030343148Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.03313683Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.035768429Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.038464248Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.041400733Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.041997897Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.044795383Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.047726317Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.050320128Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.05085562Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.053607257Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.056262937Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.058984361Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.05948155Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.0621682Z level=info msg="Executing migration" id="Move region to single row" 2024-04-05T02:49:04.111 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.064503362Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.06733279Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.067788793Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.070332536Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.073157798Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.075855721Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.076482525Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.079312032Z level=info msg="Executing migration" id="create test_data table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.081995022Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.082651607Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.083129922Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.083929964Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.084703305Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.085483676Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.085824948Z level=info msg="Executing migration" id="create team table" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.086689865Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-05T02:49:04.112 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.087616107Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.088301Z level=info msg="Executing migration" id="create team member table" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.088740988Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.089527352Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.090289168Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.0911362Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.091878486Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.09265352Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.093483025Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.094255845Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.094733036Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.095591914Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.096460105Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.097288111Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.097735646Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.098588008Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.099514657Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-05T02:49:04.113 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.100231007Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.101166768Z level=info msg="Executing migration" id="create tag table" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.101882844Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.102824835Z level=info msg="Executing migration" id="create login attempt table" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.103490452Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.104383556Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.10526874Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-05T02:49:04.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[41818]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:04.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.114 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[29558]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:04.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:04.136 INFO:teuthology.orchestra.run.smithi150.stdout:133143986205 2024-04-05T02:49:04.136 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-05T02:49:04.353 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.107168963Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-05T02:49:04.353 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.108011988Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-05T02:49:04.353 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.108951322Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-05T02:49:04.353 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.10971039Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-05T02:49:04.353 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.110754217Z level=info msg="Executing migration" id="create user auth table" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.111451068Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.112263133Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.113084924Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.113963199Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.114784089Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.115632096Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.116662009Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.117319752Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.118165011Z level=info msg="Executing migration" id="create server_lock table" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.119065965Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.119945009Z level=info msg="Executing migration" id="create user auth token table" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.122868199Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-05T02:49:04.354 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.125520689Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.128213783Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.130955249Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.133675591Z level=info msg="Executing migration" id="create cache_data table" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.136537307Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.139284036Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.142036958Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.144789566Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.145131992Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.145461581Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.147339692Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.149981593Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.150458278Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-05T02:49:04.355 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.150769825Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.151178755Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.151607178Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.152031517Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.152474942Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.153299374Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.153751732Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.154052712Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.154526805Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.154952211Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.155404613Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.155715803Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.156143415Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.156633083Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.157049979Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-05T02:49:04.356 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.157523541Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.158354264Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.15880573Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.159242016Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.162694874Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.166082202Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.166594055Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.167004227Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.167871044Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-05T02:49:04.357 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.168345084Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-05T02:49:04.358 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.168797521Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-05T02:49:04.358 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.169256805Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.169755978Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.170055806Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.170926602Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.171796322Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.172680587Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.173174301Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.173640977Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.17447056Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.175306595Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.175748408Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.176604489Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.177478209Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.177792395Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-05T02:49:04.359 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.178264787Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.18112835Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.181763454Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.182064014Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.182960645Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.185112514Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.187871786Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.190366709Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.193182449Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.19601783Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.198543592Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.201198061Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.20398979Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-05T02:49:04.360 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.206712871Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.209335378Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.211951096Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.214716449Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.217398519Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.219888873Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.222715769Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.225150141Z level=info msg="Executing migration" id="create alert_image table" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.227816058Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.230309925Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.233047859Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.233546065Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.235576445Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.235680803Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-05T02:49:04.361 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.235970722Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.238260396Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.239149856Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.239390724Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.239512292Z level=info msg="alerts found to migrate" alerts=0 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.239913328Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.240783838Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.241716891Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.242375079Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.243194627Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.243971913Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.244946127Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-05T02:49:04.362 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.245672155Z level=info msg="Executing migration" id="create data_keys table" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.246596397Z level=info msg="Executing migration" id="create secrets table" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.247227231Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.251554209Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.252539296Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.253298614Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.257493314Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.261964935Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.26265461Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.263501427Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.264446824Z level=info msg="Executing migration" id="create permission table" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.265340875Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.2660449Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.26706099Z level=info msg="Executing migration" id="create role table" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.26774379Z level=info msg="Executing migration" id="add column display_name" 2024-04-05T02:49:04.363 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.268687165Z level=info msg="Executing migration" id="add column group_name" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.26968503Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.27054245Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.271394707Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.272252656Z level=info msg="Executing migration" id="create team role table" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.273203835Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.273931314Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.274781677Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.275533894Z level=info msg="Executing migration" id="create user role table" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.276407727Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.277157007Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.278109514Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.278883657Z level=info msg="Executing migration" id="create builtin role table" 2024-04-05T02:49:04.364 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.279862883Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.280601303Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.281409904Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.282414321Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.283267729Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.284032306Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.284974431Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.285753979Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.286672617Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.287420616Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.290364938Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.292987206Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.295825Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.298493705Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-05T02:49:04.365 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.301323141Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.303989589Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.306842643Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.30951215Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.312389199Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.315146691Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.318142281Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.320591653Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.32338706Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.326193503Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.328986647Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.331888167Z level=info msg="Executing migration" id="add correlation config column" 2024-04-05T02:49:04.366 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.334504817Z level=info msg="Executing migration" id="create entity_events table" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.337219351Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.34001397Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.34012731Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.343004449Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.343204587Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.345398282Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.348017277Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.350668547Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.353239905Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-05T02:49:04.367 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.356078896Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-05T02:49:04.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.358816781Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-05T02:49:04.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.361611118Z level=info msg="Executing migration" id="Drop public config table" 2024-04-05T02:49:04.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.364275023Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-05T02:49:04.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.367040344Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-05T02:49:04.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.369833649Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.372517555Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.375232136Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.378294004Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.380792088Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.383631241Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.384303609Z level=info msg="Executing migration" id="add share column" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.385308467Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.386140392Z level=info msg="Executing migration" id="create file table" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.387046426Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.387690659Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.388593653Z level=info msg="Executing migration" id="create file_meta table" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.389303102Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-05T02:49:04.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.39020908Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.390903195Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.391719181Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.392639509Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.393332328Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.394354541Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.39516241Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.396027188Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.396748547Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.397556027Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.397953933Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.399001578Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.400030862Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.400871396Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.410792093Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.411778278Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-05T02:49:04.639 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.412499203Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.415994775Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.416914262Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.417632898Z level=info msg="Executing migration" id="create folder table" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.418482122Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.419331372Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.420142594Z level=info msg="Executing migration" id="Update folder title length" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.420974285Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=migrator t=2024-04-05T02:49:04.421818399Z level=info msg="migrations completed" performed=485 skipped=0 duration=662.60981ms 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore t=2024-04-05T02:49:04.422200322Z level=info msg="Created default organization" 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=plugin.loader t=2024-04-05T02:49:04.477944918Z level=info msg="Plugin registered" pluginID=input 2024-04-05T02:49:04.640 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=plugin.loader t=2024-04-05T02:49:04.506493738Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-05T02:49:04.641 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=plugin.loader t=2024-04-05T02:49:04.506523411Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-05T02:49:04.641 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=secrets t=2024-04-05T02:49:04.506555532Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-05T02:49:04.641 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=query_data t=2024-04-05T02:49:04.508204873Z level=info msg="Query Service initialization" 2024-04-05T02:49:04.641 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=live.push_http t=2024-04-05T02:49:04.510443611Z level=info msg="Live Push Gateway initialization" 2024-04-05T02:49:05.219 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:04 smithi196 ceph-mon[33169]: Deploying daemon node-exporter.a on smithi150 2024-04-05T02:49:05.222 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:04 smithi150 ceph-mon[41818]: Deploying daemon node-exporter.a on smithi150 2024-04-05T02:49:05.223 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:04 smithi150 ceph-mon[29558]: Deploying daemon node-exporter.a on smithi150 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=infra.usagestats.collector t=2024-04-05T02:49:05.220220147Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=provisioning.datasources t=2024-04-05T02:49:05.220588853Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=provisioning.datasources t=2024-04-05T02:49:05.254013492Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=provisioning.datasources t=2024-04-05T02:49:05.285717993Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=provisioning.alerting t=2024-04-05T02:49:05.307473322Z level=info msg="starting to provision alerting" 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=provisioning.alerting t=2024-04-05T02:49:05.307493113Z level=info msg="finished to provision alerting" 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=grafanaStorageLogger t=2024-04-05T02:49:05.307690689Z level=info msg="storage starting" 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=ngalert.state.manager t=2024-04-05T02:49:05.307762312Z level=info msg="Warming state cache for startup" 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=http.server t=2024-04-05T02:49:05.309527529Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=ngalert.state.manager t=2024-04-05T02:49:05.317911917Z level=info msg="State cache has been initialized" states=0 duration=10.149505ms 2024-04-05T02:49:05.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=ngalert.multiorg.alertmanager t=2024-04-05T02:49:05.317957582Z level=info msg="starting MultiOrg Alertmanager" 2024-04-05T02:49:05.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=ticker t=2024-04-05T02:49:05.31797309Z level=info msg=starting first_tick=2024-04-05T02:49:10Z 2024-04-05T02:49:05.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore.transactions t=2024-04-05T02:49:05.318659294Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-05T02:49:05.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore.transactions t=2024-04-05T02:49:05.357667922Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-05T02:49:05.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore.transactions t=2024-04-05T02:49:05.390648648Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-05T02:49:05.638 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[64338]: logger=sqlstore.transactions t=2024-04-05T02:49:05.401150185Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-05T02:49:06.067 INFO:teuthology.orchestra.run.smithi150.stdout:77309411367 2024-04-05T02:49:06.067 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-05T02:49:06.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:05 smithi150 ceph-mon[41818]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:06.160 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:05 smithi150 ceph-mon[29558]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:05 smithi196 ceph-mon[33169]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:06.841 INFO:teuthology.orchestra.run.smithi150.stdout:111669149731 2024-04-05T02:49:06.841 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-05T02:49:07.835 INFO:teuthology.orchestra.run.smithi150.stdout:34359738417 2024-04-05T02:49:07.836 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-05T02:49:08.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:07 smithi150 ceph-mon[41818]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:08.084 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:07 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:08.084 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:07 smithi150 ceph-mon[29558]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:08.084 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:07 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:07 smithi196 ceph-mon[33169]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:07 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:09.699 INFO:teuthology.orchestra.run.smithi150.stdout:55834574893 2024-04-05T02:49:09.699 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-05T02:49:09.926 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:09.928 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:09.936 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:09 smithi150 ceph-mon[41818]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:09.936 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:09 smithi150 ceph-mon[29558]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:09 smithi196 ceph-mon[33169]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:10.919 INFO:teuthology.orchestra.run.smithi150.stdout:201863462933 2024-04-05T02:49:10.919 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-05T02:49:11.014 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:11.020 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:11.021 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:11.417 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:11 smithi150 bash[75598]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-05T02:49:11.417 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:11 smithi150 bash[75598]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-05T02:49:11.417 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:11 smithi150 bash[75598]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-05T02:49:11.743 INFO:teuthology.orchestra.run.smithi150.stdout:236223201296 2024-04-05T02:49:11.743 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-05T02:49:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:11 smithi150 ceph-mon[29558]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:11.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:11 smithi150 ceph-mon[41818]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:11 smithi196 ceph-mon[33169]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:12.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:49:12 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:49:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:49:13.712 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:13 smithi150 bash[75598]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-05T02:49:13.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:13 smithi150 ceph-mon[29558]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:13 smithi150 ceph-mon[41818]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:13.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:13 smithi150 bash[75598]: Writing manifest to image destination 2024-04-05T02:49:14.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:13 smithi196 ceph-mon[33169]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:14.566 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:14.678 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:14 smithi150 podman[75598]: 2024-04-05 02:49:14.304648151 +0000 UTC m=+3.342589560 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T02:49:15.727 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:15 smithi150 podman[75598]: 2024-04-05 02:49:15.550624025 +0000 UTC m=+4.588565432 container create ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:16.028 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:15 smithi150 ceph-mon[29558]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:16.028 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:15 smithi150 ceph-mon[41818]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:15 smithi196 ceph-mon[33169]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:16.978 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 podman[75598]: 2024-04-05 02:49:16.90253567 +0000 UTC m=+5.940477078 container init ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:16.978 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 podman[75598]: 2024-04-05 02:49:16.908939728 +0000 UTC m=+5.946881137 container start ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:16.978 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.913Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.913Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.913Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.913Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T02:49:16.979 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T02:49:16.980 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T02:49:16.981 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.914Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.915Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.915Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.915Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T02:49:16.982 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[75991]: ts=2024-04-05T02:49:16.915Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T02:49:17.259 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:16 smithi150 bash[75598]: ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 2024-04-05T02:49:17.259 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:49:17 smithi150 systemd[1]: Started Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:18.067 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:17 smithi196 bash[64616]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-05T02:49:18.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[29558]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[29558]: Deploying daemon node-exporter.b on smithi196 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[41818]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[41818]: Deploying daemon node-exporter.b on smithi196 2024-04-05T02:49:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:18 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:18.321 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:18 smithi196 ceph-mon[33169]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:18.322 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:18 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.322 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:18 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.322 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:18 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:18.322 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:18 smithi196 ceph-mon[33169]: Deploying daemon node-exporter.b on smithi196 2024-04-05T02:49:18.322 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:18 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:18.636 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:18 smithi196 bash[64616]: Getting image source signatures 2024-04-05T02:49:18.636 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:18 smithi196 bash[64616]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-05T02:49:18.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:18 smithi196 bash[64616]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-05T02:49:18.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:18 smithi196 bash[64616]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-05T02:49:19.051 INFO:teuthology.orchestra.run.smithi150.stdout:133143986208 2024-04-05T02:49:19.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:19 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1653746155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T02:49:19.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:19 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3045892448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T02:49:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:19 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1653746155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T02:49:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:19 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3045892448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T02:49:19.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:19 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1653746155' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T02:49:19.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:19 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3045892448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T02:49:19.643 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986205 got 133143986208 for osd.4 2024-04-05T02:49:19.643 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:19.743 INFO:teuthology.orchestra.run.smithi150.stdout:167503724570 2024-04-05T02:49:20.073 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:19 smithi196 bash[64616]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-05T02:49:20.073 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:19 smithi196 bash[64616]: Writing manifest to image destination 2024-04-05T02:49:20.326 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:20.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-mon[33169]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:20.386 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 podman[64616]: 2024-04-05 02:49:20.052806942 +0000 UTC m=+2.186782681 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T02:49:20.386 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 podman[64616]: 2024-04-05 02:49:20.159656053 +0000 UTC m=+2.293631782 container create 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:49:20.421 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:20 smithi150 ceph-mon[29558]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:20.421 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:20 smithi150 ceph-mon[41818]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 podman[64616]: 2024-04-05 02:49:20.38676704 +0000 UTC m=+2.520742770 container init 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 podman[64616]: 2024-04-05 02:49:20.393354457 +0000 UTC m=+2.527330187 container start 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.397Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.397Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.397Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.397Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.398Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.398Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T02:49:20.637 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T02:49:20.638 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T02:49:20.639 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T02:49:20.640 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T02:49:20.641 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T02:49:20.641 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T02:49:20.641 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[64666]: ts=2024-04-05T02:49:20.399Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T02:49:20.641 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 bash[64616]: 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a 2024-04-05T02:49:20.641 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:49:20 smithi196 systemd[1]: Started Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[41818]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:49:21.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:21 smithi150 ceph-mon[29558]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:21.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:49:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:21 smithi196 ceph-mon[33169]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:22.123 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724570 for osd.5 2024-04-05T02:49:22.123 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:22.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:49:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:49:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:49:22.937 INFO:teuthology.orchestra.run.smithi150.stdout:111669149734 2024-04-05T02:49:23.511 INFO:teuthology.orchestra.run.smithi150.stdout:55834574896 2024-04-05T02:49:24.036 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[41818]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:24.036 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:24.036 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3352393141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T02:49:24.036 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1705931421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T02:49:24.037 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[29558]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:24.037 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:24.037 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3352393141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T02:49:24.037 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:23 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1705931421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T02:49:24.059 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149731 got 111669149734 for osd.3 2024-04-05T02:49:24.059 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:23 smithi196 ceph-mon[33169]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:23 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:23 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3352393141' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T02:49:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:23 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1705931421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T02:49:24.435 INFO:teuthology.orchestra.run.smithi150.stdout:201863462936 2024-04-05T02:49:24.458 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:25.045 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574896 for osd.1 2024-04-05T02:49:25.045 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:25.293 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:25 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:25.293 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:25 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:25.293 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:25 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2240031104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T02:49:25.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:25 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:25.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:25 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:25.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:25 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2240031104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T02:49:25.512 INFO:teuthology.orchestra.run.smithi150.stdout:34359738421 2024-04-05T02:49:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:25 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:25 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:25 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2240031104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T02:49:25.728 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462933 got 201863462936 for osd.6 2024-04-05T02:49:25.728 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:26 smithi150 ceph-mon[41818]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:26 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2126737562' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T02:49:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:26 smithi150 ceph-mon[29558]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:26 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2126737562' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T02:49:26.287 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738421 for osd.0 2024-04-05T02:49:26.287 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:26.538 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:26 smithi196 ceph-mon[33169]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:26.538 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:26 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2126737562' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T02:49:28.261 INFO:teuthology.orchestra.run.smithi150.stdout:77309411372 2024-04-05T02:49:28.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:28 smithi150 ceph-mon[41818]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:28.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:28 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1866031806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T02:49:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:28 smithi150 ceph-mon[29558]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:28 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1866031806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T02:49:28.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:28 smithi196 ceph-mon[33169]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:28.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:28 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1866031806' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T02:49:28.856 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411372 for osd.2 2024-04-05T02:49:28.856 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:30.294 INFO:teuthology.orchestra.run.smithi150.stdout:236223201299 2024-04-05T02:49:30.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:30 smithi150 ceph-mon[29558]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:30.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:30 smithi150 ceph-mon[41818]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:30.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:30 smithi196 ceph-mon[33169]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:30.960 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201296 got 236223201299 for osd.7 2024-04-05T02:49:30.960 DEBUG:teuthology.parallel:result is None 2024-04-05T02:49:30.960 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-05T02:49:30.961 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T02:49:31.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:31 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/24636679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T02:49:31.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:31 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/24636679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T02:49:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:31 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/24636679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T02:49:31.674 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:32 smithi150 ceph-mon[29558]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:32.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:32 smithi150 ceph-mon[41818]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:32.525 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:32 smithi196 ceph-mon[33169]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:32.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:49:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:49:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:49:33.252 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:33 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:33.253 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:33 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:33.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:33 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:33.789 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:49:33.791 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-04-05T02:49:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[29558]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[41818]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:49:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:34 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.505 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":37,"stamp":"2024-04-05T02:49:32.836739+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385564,"kb_used_data":6712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575652,"statfs":{"total":767960285184,"available":765517467648,"internally_reserved":0,"allocated":6873088,"data_stored":3897575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002113"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210118+0000","last_change":"2024-04-05T02:48:35.842380+0000","last_active":"2024-04-05T02:48:37.210118+0000","last_peered":"2024-04-05T02:48:37.210118+0000","last_clean":"2024-04-05T02:48:37.210118+0000","last_became_active":"2024-04-05T02:48:35.842036+0000","last_became_peered":"2024-04-05T02:48:35.842036+0000","last_unstale":"2024-04-05T02:48:37.210118+0000","last_undegraded":"2024-04-05T02:48:37.210118+0000","last_fullsized":"2024-04-05T02:48:37.210118+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:50.319270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210184+0000","last_change":"2024-04-05T02:48:36.012430+0000","last_active":"2024-04-05T02:48:37.210184+0000","last_peered":"2024-04-05T02:48:37.210184+0000","last_clean":"2024-04-05T02:48:37.210184+0000","last_became_active":"2024-04-05T02:48:36.012069+0000","last_became_peered":"2024-04-05T02:48:36.012069+0000","last_unstale":"2024-04-05T02:48:37.210184+0000","last_undegraded":"2024-04-05T02:48:37.210184+0000","last_fullsized":"2024-04-05T02:48:37.210184+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T02:59:27.856396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839880+0000","last_change":"2024-04-05T02:48:36.033701+0000","last_active":"2024-04-05T02:48:36.839880+0000","last_peered":"2024-04-05T02:48:36.839880+0000","last_clean":"2024-04-05T02:48:36.839880+0000","last_became_active":"2024-04-05T02:48:36.033383+0000","last_became_peered":"2024-04-05T02:48:36.033383+0000","last_unstale":"2024-04-05T02:48:36.839880+0000","last_undegraded":"2024-04-05T02:48:36.839880+0000","last_fullsized":"2024-04-05T02:48:36.839880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:57:58.141349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839850+0000","last_change":"2024-04-05T02:48:36.033673+0000","last_active":"2024-04-05T02:48:36.839850+0000","last_peered":"2024-04-05T02:48:36.839850+0000","last_clean":"2024-04-05T02:48:36.839850+0000","last_became_active":"2024-04-05T02:48:36.033304+0000","last_became_peered":"2024-04-05T02:48:36.033304+0000","last_unstale":"2024-04-05T02:48:36.839850+0000","last_undegraded":"2024-04-05T02:48:36.839850+0000","last_fullsized":"2024-04-05T02:48:36.839850+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:26.516492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":266,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839836+0000","last_change":"2024-04-05T02:48:04.212692+0000","last_active":"2024-04-05T02:48:36.839836+0000","last_peered":"2024-04-05T02:48:36.839836+0000","last_clean":"2024-04-05T02:48:36.839836+0000","last_became_active":"2024-04-05T02:48:04.212212+0000","last_became_peered":"2024-04-05T02:48:04.212212+0000","last_unstale":"2024-04-05T02:48:36.839836+0000","last_undegraded":"2024-04-05T02:48:36.839836+0000","last_fullsized":"2024-04-05T02:48:36.839836+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_clean_scrub_stamp":"2024-04-05T02:45:59.919732+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:33:13.395158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210159+0000","last_change":"2024-04-05T02:48:35.842260+0000","last_active":"2024-04-05T02:48:37.210159+0000","last_peered":"2024-04-05T02:48:37.210159+0000","last_clean":"2024-04-05T02:48:37.210159+0000","last_became_active":"2024-04-05T02:48:35.841968+0000","last_became_peered":"2024-04-05T02:48:35.841968+0000","last_unstale":"2024-04-05T02:48:37.210159+0000","last_undegraded":"2024-04-05T02:48:37.210159+0000","last_fullsized":"2024-04-05T02:48:37.210159+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:37:28.928023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839426+0000","last_change":"2024-04-05T02:48:36.033794+0000","last_active":"2024-04-05T02:48:36.839426+0000","last_peered":"2024-04-05T02:48:36.839426+0000","last_clean":"2024-04-05T02:48:36.839426+0000","last_became_active":"2024-04-05T02:48:36.033472+0000","last_became_peered":"2024-04-05T02:48:36.033472+0000","last_unstale":"2024-04-05T02:48:36.839426+0000","last_undegraded":"2024-04-05T02:48:36.839426+0000","last_fullsized":"2024-04-05T02:48:36.839426+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:39:42.977557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210104+0000","last_change":"2024-04-05T02:48:35.840743+0000","last_active":"2024-04-05T02:48:37.210104+0000","last_peered":"2024-04-05T02:48:37.210104+0000","last_clean":"2024-04-05T02:48:37.210104+0000","last_became_active":"2024-04-05T02:48:35.840424+0000","last_became_peered":"2024-04-05T02:48:35.840424+0000","last_unstale":"2024-04-05T02:48:37.210104+0000","last_undegraded":"2024-04-05T02:48:37.210104+0000","last_fullsized":"2024-04-05T02:48:37.210104+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:04:16.216370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210182+0000","last_change":"2024-04-05T02:48:36.034060+0000","last_active":"2024-04-05T02:48:37.210182+0000","last_peered":"2024-04-05T02:48:37.210182+0000","last_clean":"2024-04-05T02:48:37.210182+0000","last_became_active":"2024-04-05T02:48:36.033747+0000","last_became_peered":"2024-04-05T02:48:36.033747+0000","last_unstale":"2024-04-05T02:48:37.210182+0000","last_undegraded":"2024-04-05T02:48:37.210182+0000","last_fullsized":"2024-04-05T02:48:37.210182+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:16:53.864447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201300,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.45800000000000002}]},{"osd":1,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49299999999999999}]},{"osd":2,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.48399999999999999}]},{"osd":3,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.441}]},{"osd":4,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.44700000000000001}]},{"osd":5,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.376},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.40500000000000003}]},{"osd":6,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.42399999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.753},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.64200000000000002}]},{"osd":1,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.77100000000000002}]},{"osd":2,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.79400000000000004}]},{"osd":3,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.73699999999999999}]},{"osd":4,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"last":0.501},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.67900000000000005}]},{"osd":5,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.65700000000000003}]},{"osd":7,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.624},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.71099999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.35099999999999998,"15min":0.32100000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.53600000000000003,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.375,"15min":0.34200000000000003},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.66600000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.48999999999999999}]},{"osd":2,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.376,"15min":0.373},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.56200000000000006,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.40799999999999997,"15min":0.39400000000000002},"min":{"1min":0.27700000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.45900000000000002}]},{"osd":3,"last update":"Fri Apr 5 02:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.49099999999999999,"15min":0.48899999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.437,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.50600000000000001}]},{"osd":4,"last update":"Fri Apr 5 02:48:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.622,"15min":0.63100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79800000000000004,"5min":1.022,"15min":1.022},"last":0.626},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.58199999999999996,"15min":0.59099999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.66400000000000003}]},{"osd":5,"last update":"Fri Apr 5 02:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.56699999999999995,"15min":0.57499999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.60699999999999998}]},{"osd":6,"last update":"Fri Apr 5 02:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.58999999999999997}]},{"osd":7,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.69399999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.33500000000000002,"15min":0.31900000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.36499999999999999,"15min":0.34300000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56799999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.38400000000000001}]},{"osd":2,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.42699999999999999,"15min":0.441},"min":{"1min":0.30199999999999999,"5min":0.246,"15min":0.246},"max":{"1min":0.58199999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40400000000000003,"15min":0.40100000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.36799999999999999}]},{"osd":3,"last update":"Fri Apr 5 02:49:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.61599999999999999,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64200000000000002,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.40300000000000002}]},{"osd":4,"last update":"Fri Apr 5 02:48:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.54500000000000004,"15min":0.53600000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.439}]},{"osd":5,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.502},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.59699999999999998,"15min":0.60199999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.878,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.495}]},{"osd":6,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.51200000000000001}]},{"osd":7,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.51800000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":265989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.25700000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.53200000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.55300000000000005,"5min":0.62,"15min":0.62},"last":0.39100000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.41099999999999998,"15min":0.39900000000000002},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.56699999999999995,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.496},{"interface":"front","average":{"1min":0.442,"5min":0.44900000000000001,"15min":0.42899999999999999},"min":{"1min":0.32600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.64500000000000002,"5min":0.751,"15min":0.751},"last":0.40200000000000002}]},{"osd":3,"last update":"Fri Apr 5 02:48:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.50900000000000001,"15min":0.50700000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73699999999999999,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.46500000000000002,"15min":0.46999999999999997},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.62,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.51200000000000001}]},{"osd":4,"last update":"Fri Apr 5 02:48:48 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.60799999999999998,"15min":0.60599999999999998},"min":{"1min":0.45100000000000001,"5min":0.436,"15min":0.436},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82999999999999996,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.74099999999999999}]},{"osd":5,"last update":"Fri Apr 5 02:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.57799999999999996,"15min":0.58599999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.745,"15min":0.745},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.63500000000000001,"15min":0.64100000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.85599999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.66700000000000004}]},{"osd":6,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.70499999999999996}]},{"osd":7,"last update":"Fri Apr 5 02:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.71699999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.39600000000000002,"15min":0.39200000000000002},"min":{"1min":0.28899999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.42399999999999999,"15min":0.41599999999999998},"min":{"1min":0.34300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.25900000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.61599999999999999,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.436,"15min":0.433},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.32500000000000001}]},{"osd":2,"last update":"Fri Apr 5 02:48:26 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.46000000000000002,"15min":0.45400000000000001},"min":{"1min":0.39900000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.317}]},{"osd":4,"last update":"Fri Apr 5 02:48:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.41999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.625,"5min":0.58699999999999997,"15min":0.58099999999999996},"min":{"1min":0.44800000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.58499999999999996}]},{"osd":5,"last update":"Fri Apr 5 02:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.60799999999999998,"15min":0.60399999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.59299999999999997,"15min":0.58899999999999997},"min":{"1min":0.46300000000000002,"5min":0.443,"15min":0.443},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.48799999999999999}]},{"osd":6,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.53900000000000003}]},{"osd":7,"last update":"Fri Apr 5 02:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.51900000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":265989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.50600000000000001,"15min":0.504},"min":{"1min":0.36899999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.503,"15min":0.495},"min":{"1min":0.34799999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.50700000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.41899999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.52400000000000002,"15min":0.51800000000000002},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.38800000000000001}]},{"osd":2,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.505,"15min":0.497},"min":{"1min":0.35799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.437},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.377,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73099999999999998,"5min":0.747,"15min":0.747},"last":0.51500000000000001}]},{"osd":3,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.54500000000000004,"15min":0.53900000000000003},"min":{"1min":0.43099999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.40500000000000003}]},{"osd":5,"last update":"Fri Apr 5 02:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.42499999999999999,"15min":0.41899999999999998},"min":{"1min":0.26400000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73199999999999998,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.42199999999999999,"15min":0.40799999999999997},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.33100000000000002}]},{"osd":6,"last update":"Fri Apr 5 02:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.52600000000000002}]},{"osd":7,"last update":"Fri Apr 5 02:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.496}]}]},{"osd":5,"up_from":39,"seq":167503724573,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.55000000000000004,"15min":0.55900000000000005},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.73599999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.52600000000000002,"15min":0.53600000000000003},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64700000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.35999999999999999}]},{"osd":1,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71199999999999997,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.371},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68100000000000005,"5min":0.745,"15min":0.745},"last":0.437}]},{"osd":2,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.52700000000000002,"15min":0.53300000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.59799999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.378},{"interface":"front","average":{"1min":0.505,"5min":0.56599999999999995,"15min":0.57599999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72699999999999998,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.42699999999999999}]},{"osd":3,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.52500000000000002,"15min":0.52700000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.63300000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71499999999999997,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.41699999999999998}]},{"osd":4,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.44,"15min":0.432},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.69399999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.439,"15min":0.42999999999999999},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63900000000000001,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.34300000000000003}]},{"osd":6,"last update":"Fri Apr 5 02:48:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.316},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.30299999999999999}]},{"osd":7,"last update":"Fri Apr 5 02:49:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.40600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T02:49:34.507 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T02:49:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:34 smithi196 ceph-mon[33169]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:34 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:34 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:34 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:34 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:49:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:34 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:35.398 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:35 smithi150 ceph-mon[29558]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:49:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:35 smithi150 ceph-mon[29558]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T02:49:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:35 smithi150 ceph-mon[29558]: Reconfiguring daemon alertmanager.a on smithi150 2024-04-05T02:49:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:35 smithi150 ceph-mon[41818]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:49:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:35 smithi150 ceph-mon[41818]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T02:49:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:35 smithi150 ceph-mon[41818]: Reconfiguring daemon alertmanager.a on smithi150 2024-04-05T02:49:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:35 smithi196 ceph-mon[33169]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:49:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:35 smithi196 ceph-mon[33169]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T02:49:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:35 smithi196 ceph-mon[33169]: Reconfiguring daemon alertmanager.a on smithi150 2024-04-05T02:49:36.207 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:36 smithi150 ceph-mon[29558]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:36.468 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:36 smithi150 ceph-mon[41818]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:36 smithi196 ceph-mon[33169]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:37.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:37 smithi150 ceph-mon[29558]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:37.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:37 smithi150 ceph-mon[41818]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:37 smithi196 ceph-mon[33169]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:38.318 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:49:38.319 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-04-05T02:49:39.050 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":39,"stamp":"2024-04-05T02:49:36.837309+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385564,"kb_used_data":6712,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575652,"statfs":{"total":767960285184,"available":765517467648,"internally_reserved":0,"allocated":6873088,"data_stored":3897575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002126"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210118+0000","last_change":"2024-04-05T02:48:35.842380+0000","last_active":"2024-04-05T02:48:37.210118+0000","last_peered":"2024-04-05T02:48:37.210118+0000","last_clean":"2024-04-05T02:48:37.210118+0000","last_became_active":"2024-04-05T02:48:35.842036+0000","last_became_peered":"2024-04-05T02:48:35.842036+0000","last_unstale":"2024-04-05T02:48:37.210118+0000","last_undegraded":"2024-04-05T02:48:37.210118+0000","last_fullsized":"2024-04-05T02:48:37.210118+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:50.319270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210184+0000","last_change":"2024-04-05T02:48:36.012430+0000","last_active":"2024-04-05T02:48:37.210184+0000","last_peered":"2024-04-05T02:48:37.210184+0000","last_clean":"2024-04-05T02:48:37.210184+0000","last_became_active":"2024-04-05T02:48:36.012069+0000","last_became_peered":"2024-04-05T02:48:36.012069+0000","last_unstale":"2024-04-05T02:48:37.210184+0000","last_undegraded":"2024-04-05T02:48:37.210184+0000","last_fullsized":"2024-04-05T02:48:37.210184+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T02:59:27.856396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839880+0000","last_change":"2024-04-05T02:48:36.033701+0000","last_active":"2024-04-05T02:48:36.839880+0000","last_peered":"2024-04-05T02:48:36.839880+0000","last_clean":"2024-04-05T02:48:36.839880+0000","last_became_active":"2024-04-05T02:48:36.033383+0000","last_became_peered":"2024-04-05T02:48:36.033383+0000","last_unstale":"2024-04-05T02:48:36.839880+0000","last_undegraded":"2024-04-05T02:48:36.839880+0000","last_fullsized":"2024-04-05T02:48:36.839880+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:57:58.141349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839850+0000","last_change":"2024-04-05T02:48:36.033673+0000","last_active":"2024-04-05T02:48:36.839850+0000","last_peered":"2024-04-05T02:48:36.839850+0000","last_clean":"2024-04-05T02:48:36.839850+0000","last_became_active":"2024-04-05T02:48:36.033304+0000","last_became_peered":"2024-04-05T02:48:36.033304+0000","last_unstale":"2024-04-05T02:48:36.839850+0000","last_undegraded":"2024-04-05T02:48:36.839850+0000","last_fullsized":"2024-04-05T02:48:36.839850+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:26.516492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":266,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839836+0000","last_change":"2024-04-05T02:48:04.212692+0000","last_active":"2024-04-05T02:48:36.839836+0000","last_peered":"2024-04-05T02:48:36.839836+0000","last_clean":"2024-04-05T02:48:36.839836+0000","last_became_active":"2024-04-05T02:48:04.212212+0000","last_became_peered":"2024-04-05T02:48:04.212212+0000","last_unstale":"2024-04-05T02:48:36.839836+0000","last_undegraded":"2024-04-05T02:48:36.839836+0000","last_fullsized":"2024-04-05T02:48:36.839836+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_clean_scrub_stamp":"2024-04-05T02:45:59.919732+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:33:13.395158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210159+0000","last_change":"2024-04-05T02:48:35.842260+0000","last_active":"2024-04-05T02:48:37.210159+0000","last_peered":"2024-04-05T02:48:37.210159+0000","last_clean":"2024-04-05T02:48:37.210159+0000","last_became_active":"2024-04-05T02:48:35.841968+0000","last_became_peered":"2024-04-05T02:48:35.841968+0000","last_unstale":"2024-04-05T02:48:37.210159+0000","last_undegraded":"2024-04-05T02:48:37.210159+0000","last_fullsized":"2024-04-05T02:48:37.210159+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:37:28.928023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:36.839426+0000","last_change":"2024-04-05T02:48:36.033794+0000","last_active":"2024-04-05T02:48:36.839426+0000","last_peered":"2024-04-05T02:48:36.839426+0000","last_clean":"2024-04-05T02:48:36.839426+0000","last_became_active":"2024-04-05T02:48:36.033472+0000","last_became_peered":"2024-04-05T02:48:36.033472+0000","last_unstale":"2024-04-05T02:48:36.839426+0000","last_undegraded":"2024-04-05T02:48:36.839426+0000","last_fullsized":"2024-04-05T02:48:36.839426+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:39:42.977557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210104+0000","last_change":"2024-04-05T02:48:35.840743+0000","last_active":"2024-04-05T02:48:37.210104+0000","last_peered":"2024-04-05T02:48:37.210104+0000","last_clean":"2024-04-05T02:48:37.210104+0000","last_became_active":"2024-04-05T02:48:35.840424+0000","last_became_peered":"2024-04-05T02:48:35.840424+0000","last_unstale":"2024-04-05T02:48:37.210104+0000","last_undegraded":"2024-04-05T02:48:37.210104+0000","last_fullsized":"2024-04-05T02:48:37.210104+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:04:16.216370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-05T02:48:37.210182+0000","last_change":"2024-04-05T02:48:36.034060+0000","last_active":"2024-04-05T02:48:37.210182+0000","last_peered":"2024-04-05T02:48:37.210182+0000","last_clean":"2024-04-05T02:48:37.210182+0000","last_became_active":"2024-04-05T02:48:36.033747+0000","last_became_peered":"2024-04-05T02:48:36.033747+0000","last_unstale":"2024-04-05T02:48:37.210182+0000","last_undegraded":"2024-04-05T02:48:37.210182+0000","last_fullsized":"2024-04-05T02:48:37.210182+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:16:53.864447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.74099999999999999}]},{"osd":1,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.78000000000000003}]},{"osd":2,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.79500000000000004}]},{"osd":3,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.877}]},{"osd":4,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.76200000000000001}]},{"osd":5,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.83699999999999997},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.82199999999999995}]},{"osd":6,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.85999999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.81000000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.52500000000000002}]},{"osd":1,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.55900000000000005}]},{"osd":2,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57899999999999996}]},{"osd":3,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.55200000000000005}]},{"osd":4,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.45500000000000002}]},{"osd":5,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.625}]},{"osd":7,"last update":"Fri Apr 5 02:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.503},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.53700000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.35099999999999998,"15min":0.32100000000000001},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":0.53600000000000003,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.375,"15min":0.34200000000000003},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.66600000000000004,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.34200000000000003}]},{"osd":2,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.376,"15min":0.373},"min":{"1min":0.22900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.56200000000000006,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.40799999999999997,"15min":0.39400000000000002},"min":{"1min":0.27700000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.41299999999999998}]},{"osd":3,"last update":"Fri Apr 5 02:49:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.47499999999999998,"15min":0.48299999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.624,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.38,"5min":0.437,"15min":0.44900000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.56699999999999995,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.36499999999999999}]},{"osd":4,"last update":"Fri Apr 5 02:48:48 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.622,"15min":0.63100000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.79800000000000004,"5min":1.022,"15min":1.022},"last":0.623},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.58199999999999996,"15min":0.59099999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.5}]},{"osd":5,"last update":"Fri Apr 5 02:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.56699999999999995,"15min":0.57499999999999996},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.51100000000000001}]},{"osd":6,"last update":"Fri Apr 5 02:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.48699999999999999}]},{"osd":7,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.56299999999999994}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.33500000000000002,"15min":0.31900000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.36499999999999999,"15min":0.34300000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.56799999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.34100000000000003}]},{"osd":2,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.42699999999999999,"15min":0.441},"min":{"1min":0.30199999999999999,"5min":0.246,"15min":0.246},"max":{"1min":0.58199999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.40400000000000003,"15min":0.40100000000000002},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.35699999999999998}]},{"osd":3,"last update":"Fri Apr 5 02:49:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.61599999999999999,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64200000000000002,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.377}]},{"osd":4,"last update":"Fri Apr 5 02:48:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.875,"5min":0.875,"15min":0.875},"last":0.623},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.54500000000000004,"15min":0.53600000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.56100000000000005}]},{"osd":5,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59899999999999998,"15min":0.60099999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.83399999999999996,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.59699999999999998,"15min":0.60199999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.878,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.59299999999999997}]},{"osd":6,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.56699999999999995}]},{"osd":7,"last update":"Fri Apr 5 02:49:00 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.1639999999999999,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.60899999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":265989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.25700000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.53200000000000003,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.25900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.55300000000000005,"5min":0.62,"15min":0.62},"last":0.32800000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.41099999999999998,"15min":0.39900000000000002},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.56699999999999995,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.442,"5min":0.44900000000000001,"15min":0.42899999999999999},"min":{"1min":0.32600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.64500000000000002,"5min":0.751,"15min":0.751},"last":0.371}]},{"osd":3,"last update":"Fri Apr 5 02:49:32 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.495,"15min":0.503},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.60799999999999998,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.35099999999999998,"5min":0.44,"15min":0.46200000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55000000000000004,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.318}]},{"osd":4,"last update":"Fri Apr 5 02:48:48 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.60799999999999998,"15min":0.60599999999999998},"min":{"1min":0.45100000000000001,"5min":0.436,"15min":0.436},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.60999999999999999,"15min":0.62},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.82999999999999996,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.50900000000000001}]},{"osd":5,"last update":"Fri Apr 5 02:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.57799999999999996,"15min":0.58599999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.745,"15min":0.745},"last":0.442},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.63500000000000001,"15min":0.64100000000000001},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.85599999999999998,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.49299999999999999}]},{"osd":6,"last update":"Fri Apr 5 02:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.53900000000000003}]},{"osd":7,"last update":"Fri Apr 5 02:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.47699999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.40799999999999997,"15min":0.39600000000000002},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.252,"5min":1.252,"15min":1.252},"last":0.246},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.41799999999999998,"15min":0.41399999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.58899999999999997,"5min":0.63,"15min":0.63},"last":0.36099999999999999}]},{"osd":1,"last update":"Fri Apr 5 02:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42999999999999999,"15min":0.42799999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.44500000000000001,"15min":0.436},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.28100000000000003}]},{"osd":2,"last update":"Fri Apr 5 02:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.46500000000000002,"15min":0.45600000000000002},"min":{"1min":0.34799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.45700000000000002,"15min":0.45800000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.32600000000000001}]},{"osd":4,"last update":"Fri Apr 5 02:48:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.41999999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.58699999999999997,"15min":0.58099999999999996},"min":{"1min":0.44800000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0069999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.60899999999999999}]},{"osd":5,"last update":"Fri Apr 5 02:49:11 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.60799999999999998,"15min":0.60399999999999998},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.59299999999999997,"15min":0.58899999999999997},"min":{"1min":0.46300000000000002,"5min":0.443,"15min":0.443},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.747}]},{"osd":6,"last update":"Fri Apr 5 02:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.73399999999999999}]},{"osd":7,"last update":"Fri Apr 5 02:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.58299999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297984,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447168,"statfs":{"total":95995035648,"available":95689900032,"internally_reserved":0,"allocated":638976,"data_stored":265989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.50600000000000001,"15min":0.504},"min":{"1min":0.36899999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.497},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.503,"15min":0.495},"min":{"1min":0.34799999999999998,"5min":0.247,"15min":0.247},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.51300000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.498,"15min":0.48999999999999999},"min":{"1min":0.41899999999999998,"5min":0.248,"15min":0.248},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.52400000000000002,"15min":0.51800000000000002},"min":{"1min":0.38400000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.432}]},{"osd":2,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.505,"15min":0.497},"min":{"1min":0.35799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.377,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73099999999999998,"5min":0.747,"15min":0.747},"last":0.40100000000000002}]},{"osd":3,"last update":"Fri Apr 5 02:48:51 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.54500000000000004,"15min":0.53900000000000003},"min":{"1min":0.43099999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.40300000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59799999999999998}]},{"osd":5,"last update":"Fri Apr 5 02:49:14 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.42499999999999999,"15min":0.41899999999999998},"min":{"1min":0.26400000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73199999999999998,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.42199999999999999,"15min":0.40799999999999997},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.46500000000000002}]},{"osd":6,"last update":"Fri Apr 5 02:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.53900000000000003}]},{"osd":7,"last update":"Fri Apr 5 02:49:02 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.63700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.55000000000000004,"15min":0.55900000000000005},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.73599999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.52600000000000002,"15min":0.53600000000000003},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.64700000000000002,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.59099999999999997}]},{"osd":1,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.71199999999999997,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.68100000000000005,"5min":0.745,"15min":0.745},"last":0.628}]},{"osd":2,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.52700000000000002,"15min":0.53300000000000003},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.59799999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.56599999999999995,"15min":0.57599999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72699999999999998,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.55000000000000004}]},{"osd":3,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.52500000000000002,"15min":0.52700000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.63300000000000001,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71499999999999997,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.70199999999999996}]},{"osd":4,"last update":"Fri Apr 5 02:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.44,"15min":0.432},"min":{"1min":0.29899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.69399999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.439,"15min":0.42999999999999999},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63900000000000001,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.57499999999999996}]},{"osd":6,"last update":"Fri Apr 5 02:48:39 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.746},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.67300000000000004}]},{"osd":7,"last update":"Fri Apr 5 02:49:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.69199999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T02:49:39.052 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-05T02:49:39.052 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-05T02:49:39.052 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-05T02:49:39.052 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T02:49:39.747 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:39.869 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:39 smithi150 systemd[1]: Stopping Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:49:40.141 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:39 smithi150 ceph-mon[29558]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:49:40.141 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:39 smithi150 ceph-mon[29558]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:40.141 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:39 smithi150 ceph-mon[41818]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:49:40.141 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:39 smithi150 ceph-mon[41818]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:40.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:39 smithi196 ceph-mon[33169]: from='client.14607 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:49:40.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:39 smithi196 ceph-mon[33169]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:40.854 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[73041]: ts=2024-04-05T02:49:40.565Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-05T02:49:40.854 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:40 smithi150 podman[78268]: 2024-04-05 02:49:40.596390372 +0000 UTC m=+0.563676488 container died 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:41.196 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:41 smithi150 podman[78268]: 2024-04-05 02:49:41.021170532 +0000 UTC m=+0.988456655 container cleanup 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:41.197 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:41 smithi150 bash[78268]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-05T02:49:41.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:41 smithi150 podman[78306]: 2024-04-05 02:49:41.19736154 +0000 UTC m=+0.601900193 container remove 7d768a50aa234fbb2ac5a8527abd50768fb8b78906fef59889508425940c6d1e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:41.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:41 smithi150 podman[78306]: 2024-04-05 02:49:41.289719547 +0000 UTC m=+0.694258199 volume remove 07a3d78b3d188aa3a802ec553f3cc6bda517798481305b7b9ad60044d13a36a2 2024-04-05T02:49:41.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:41 smithi150 ceph-mon[29558]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:41.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:41 smithi150 ceph-mon[41818]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:41 smithi196 ceph-mon[33169]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:42.472 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:49:42.472 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-05T02:49:42.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:42 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-05T02:49:42.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:42 smithi150 systemd[1]: Stopped Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:42.728 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:42 smithi150 systemd[1]: Starting Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:49:42.728 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:42 smithi150 podman[78482]: 2024-04-05 02:49:42.433627616 +0000 UTC m=+0.017864840 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-05T02:49:42.728 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:42 smithi150 podman[78482]: 2024-04-05 02:49:42.538457096 +0000 UTC m=+0.122694300 volume create 8ad84c32e514402ebb9a0c2a5ad2e705b3702f6a337865747ef4c42f1ad6f685 2024-04-05T02:49:42.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:49:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:49:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:49:43.183 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:42 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3623254512' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T02:49:43.183 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:42 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3623254512' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T02:49:43.183 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:42 smithi150 podman[78482]: 2024-04-05 02:49:42.856412718 +0000 UTC m=+0.440649922 container create 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:43.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:42 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3623254512' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T02:49:43.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 podman[78482]: 2024-04-05 02:49:43.184085719 +0000 UTC m=+0.768322928 container init 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:43.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 podman[78482]: 2024-04-05 02:49:43.189600955 +0000 UTC m=+0.773838159 container start 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.202Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.202Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.203Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.150 port=9094 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.204Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.227Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.228Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.230Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:43.230Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 bash[78482]: 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec 2024-04-05T02:49:43.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:43 smithi150 systemd[1]: Started Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:43.710 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-05T02:49:43.710 INFO:tasks.cephadm:Setup complete, yielding 2024-04-05T02:49:43.710 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-05T02:49:43.722 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi150.front.sepia.ceph.com 2024-04-05T02:49:43.723 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- bash -c 'ceph fs volume create foo' 2024-04-05T02:49:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-mon[41818]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-mon[29558]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:43 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:43.980 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:43.985 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:43 smithi196 ceph-mon[33169]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:43.985 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:43 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:43.985 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:43 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:45.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:44 smithi150 ceph-mon[41818]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:49:45.160 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:44 smithi150 ceph-mon[41818]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:49:45.160 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:44 smithi150 ceph-mon[29558]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:49:45.160 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:44 smithi150 ceph-mon[29558]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:49:45.296 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:44 smithi196 ceph-mon[33169]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:49:45.296 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:44 smithi196 ceph-mon[33169]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:49:45.445 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:45 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:45.204Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000287713s 2024-04-05T02:49:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-mon[33169]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:46.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:46 smithi150 ceph-mon[29558]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:46 smithi150 ceph-mon[41818]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:46.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 systemd[1]: Stopping Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:49:47.252 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:49:47.253 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T02:49:47.253 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T02:49:47.253 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-05T02:49:47.253 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: osdmap e65: 8 total, 8 up, 8 in 2024-04-05T02:49:47.253 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:47.253 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:47 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:47.253 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T02:49:47.253 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T02:49:47.253 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T02:49:47.253 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[62143]: ts=2024-04-05T02:49:46.938Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T02:49:47.254 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:46 smithi196 podman[65557]: 2024-04-05 02:49:46.948812329 +0000 UTC m=+0.116797053 container died 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: osdmap e65: 8 total, 8 up, 8 in 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:49:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T02:49:47.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-05T02:49:47.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-05T02:49:47.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: osdmap e65: 8 total, 8 up, 8 in 2024-04-05T02:49:47.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:47.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:47.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[29535]: 2024-04-05T02:49:47.189+0000 7f6340a73700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T02:49:47.569 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 podman[65557]: 2024-04-05 02:49:47.360660092 +0000 UTC m=+0.528644850 container cleanup 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:47.569 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 bash[65557]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T02:49:47.886 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 podman[65577]: 2024-04-05 02:49:47.570312209 +0000 UTC m=+0.625156041 container remove 3d543df4692dc6e2c14c8136d86d01f25ccc84a4e3e913e8aec8507e1ccc8731 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:47.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-05T02:49:47.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 systemd[1]: Stopped Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:47.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service: Consumed 1.101s CPU time. 2024-04-05T02:49:47.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 systemd[1]: Starting Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:49:48.148 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 podman[65671]: 2024-04-05 02:49:47.794830371 +0000 UTC m=+0.018889338 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T02:49:48.148 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:47 smithi196 podman[65671]: 2024-04-05 02:49:47.946460825 +0000 UTC m=+0.170519824 container create 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:48.148 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: pgmap v45: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:48.148 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:49:48.148 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-05T02:49:48.148 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: osdmap e66: 8 total, 8 up, 8 in 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: osdmap e67: 8 total, 8 up, 8 in 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: fsmap foo:0 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:48.149 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:48.245 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: pgmap v45: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: osdmap e66: 8 total, 8 up, 8 in 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: osdmap e67: 8 total, 8 up, 8 in 2024-04-05T02:49:48.452 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: fsmap foo:0 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: pgmap v45: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: osdmap e66: 8 total, 8 up, 8 in 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-05T02:49:48.453 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: osdmap e67: 8 total, 8 up, 8 in 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: fsmap foo:0 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STOPPING 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STOPPED 2024-04-05T02:49:48.454 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STARTING 2024-04-05T02:49:48.455 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Serving on http://:::9283 2024-04-05T02:49:48.455 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STARTED 2024-04-05T02:49:48.455 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STOPPING 2024-04-05T02:49:48.537 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 podman[65671]: 2024-04-05 02:49:48.148323186 +0000 UTC m=+0.372382143 container init 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 podman[65671]: 2024-04-05 02:49:48.153635042 +0000 UTC m=+0.377694000 container start 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.177Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.177Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.177Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi196 (none))" 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.177Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T02:49:48.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.177Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.178Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.179Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.180Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.180Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.182Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.182Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.778µs 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.182Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.182Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.183Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.183Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.304µs wal_replay_duration=445.689µs wbl_replay_duration=98ns total_replay_duration=485.822µs 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.184Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.184Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.184Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.198Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.701733ms db_storage=701ns remote_storage=1.109µs web_handler=426ns query_engine=430ns scrape=567.307µs scrape_sd=22.623µs notify=20.307µs notify_sd=8.257µs rules=13.884046ms tracing=7.058µs 2024-04-05T02:49:48.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.198Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T02:49:48.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:49:48.198Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T02:49:48.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 bash[65671]: 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 2024-04-05T02:49:48.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:49:48 smithi196 systemd[1]: Started Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:49:48.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STOPPED 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STARTING 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Serving on http://:::9283 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STARTED 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STOPPING 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STOPPED 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STARTING 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Serving on http://:::9283 2024-04-05T02:49:48.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:49:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:49:48] ENGINE Bus STARTED 2024-04-05T02:49:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: Saving service mds.foo spec with placement count:2 2024-04-05T02:49:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: osdmap e68: 8 total, 8 up, 8 in 2024-04-05T02:49:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:49:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi150.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi196.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi196.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:49 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: Saving service mds.foo spec with placement count:2 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: osdmap e68: 8 total, 8 up, 8 in 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi150.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi196.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi196.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: Saving service mds.foo spec with placement count:2 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: osdmap e68: 8 total, 8 up, 8 in 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:49:49.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi150.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi196.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi196.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:49.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:49 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi150.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi196.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi196.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: osdmap e69: 8 total, 8 up, 8 in 2024-04-05T02:49:50.329 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[29558]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi150.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi196.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi196.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: osdmap e69: 8 total, 8 up, 8 in 2024-04-05T02:49:50.330 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:50 smithi150 ceph-mon[41818]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi150.front.sepia.ceph.com:9093"}]: dispatch 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi196.front.sepia.ceph.com:9095"}]: dispatch 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi196.front.sepia.ceph.com:3000"}]: dispatch 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: pgmap v49: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: osdmap e69: 8 total, 8 up, 8 in 2024-04-05T02:49:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:50 smithi196 ceph-mon[33169]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:49:51.480 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-05T02:49:51.768 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:52 smithi150 ceph-mon[29558]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:52 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:52 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:52.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:52 smithi150 ceph-mon[41818]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:52.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:52 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:52.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:52 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:52 smithi196 ceph-mon[33169]: pgmap v51: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:52 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:52 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:53.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:49:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:49:53.207Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003196536s 2024-04-05T02:49:54.318 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:54 smithi150 ceph-mon[41818]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:54.319 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:54 smithi150 ceph-mon[29558]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:54 smithi196 ceph-mon[33169]: pgmap v52: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:54.985 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-05T02:49:54.995 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-05T02:49:54.995 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-05T02:49:54.995 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: Deploying daemon mds.foo.smithi150.jnrwgs on smithi150 2024-04-05T02:49:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:55 smithi196 ceph-mon[33169]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:49:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: Deploying daemon mds.foo.smithi150.jnrwgs on smithi150 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[41818]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:55.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:55.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T02:49:55.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:55.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: Deploying daemon mds.foo.smithi150.jnrwgs on smithi150 2024-04-05T02:49:55.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:55 smithi150 ceph-mon[29558]: pgmap v53: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:55.730 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:49:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:57 smithi150 ceph-mon[29558]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:57.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:57 smithi150 ceph-mon[41818]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:58.340 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:57 smithi196 ceph-mon[33169]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:49:58.384 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:49:59.124 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2007018875' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T02:49:59.124 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.124 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.124 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.124 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:59.124 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2007018875' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T02:49:59.125 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:58 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:59.149 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":20,"active_gid":14427,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":1834837291},{"type":"v1","addr":"172.21.15.150:6801","nonce":1834837291}]},"active_addr":"172.21.15.150:6801/1834837291","active_change":"2024-04-05T02:48:32.814886+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24335,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.150:8443/","prometheus":"http://172.21.15.150:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":11242322}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":198010180}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2745855992}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2603883274}]}]} 2024-04-05T02:49:59.154 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-05T02:49:59.154 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-05T02:49:59.154 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2007018875' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-05T02:49:59.181 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:58 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:49:59.412 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:00.136 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: Deploying daemon mds.foo.smithi196.eohomm on smithi196 2024-04-05T02:50:00.136 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:00.136 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:boot 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: daemon mds.foo.smithi150.jnrwgs assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: Cluster is now healthy 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: fsmap foo:0 1 up:standby 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:creating} 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[41818]: daemon mds.foo.smithi150.jnrwgs is now active in filesystem foo as rank 0 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: Deploying daemon mds.foo.smithi196.eohomm on smithi196 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:boot 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: daemon mds.foo.smithi150.jnrwgs assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-05T02:50:00.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-05T02:50:00.138 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: Cluster is now healthy 2024-04-05T02:50:00.138 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:50:00.138 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: fsmap foo:0 1 up:standby 2024-04-05T02:50:00.138 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:creating} 2024-04-05T02:50:00.138 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:49:59 smithi150 ceph-mon[29558]: daemon mds.foo.smithi150.jnrwgs is now active in filesystem foo as rank 0 2024-04-05T02:50:00.159 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: Deploying daemon mds.foo.smithi196.eohomm on smithi196 2024-04-05T02:50:00.159 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: pgmap v55: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:00.159 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:boot 2024-04-05T02:50:00.159 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: daemon mds.foo.smithi150.jnrwgs assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-05T02:50:00.159 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-05T02:50:00.160 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-05T02:50:00.160 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: Cluster is now healthy 2024-04-05T02:50:00.160 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:50:00.160 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: fsmap foo:0 1 up:standby 2024-04-05T02:50:00.160 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:creating} 2024-04-05T02:50:00.160 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:49:59 smithi196 ceph-mon[33169]: daemon mds.foo.smithi150.jnrwgs is now active in filesystem foo as rank 0 2024-04-05T02:50:00.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:00] "GET /metrics HTTP/1.1" 200 208132 "" "Prometheus/2.43.0" 2024-04-05T02:50:01.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:00 smithi196 ceph-mon[33169]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:active 2024-04-05T02:50:01.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:00 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 2024-04-05T02:50:01.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:00 smithi196 ceph-mon[33169]: overall HEALTH_OK 2024-04-05T02:50:01.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-mon[41818]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:active 2024-04-05T02:50:01.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 2024-04-05T02:50:01.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-mon[41818]: overall HEALTH_OK 2024-04-05T02:50:01.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-mon[29558]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:active 2024-04-05T02:50:01.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-mon[29558]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 2024-04-05T02:50:01.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:00 smithi150 ceph-mon[29558]: overall HEALTH_OK 2024-04-05T02:50:01.312 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:50:01.312 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":69,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T02:49:49.199260+0000","last_up_change":"2024-04-05T02:48:00.713775+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T02:48:34.437290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T02:49:46.120483+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T02:49:46.414509+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6803","nonce":4116662696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6805","nonce":4116662696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6809","nonce":4116662696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6807","nonce":4116662696}]},"public_addr":"172.21.15.150:6803/4116662696","cluster_addr":"172.21.15.150:6805/4116662696","heartbeat_back_addr":"172.21.15.150:6809/4116662696","heartbeat_front_addr":"172.21.15.150:6807/4116662696","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6811","nonce":3820311304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6813","nonce":3820311304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6817","nonce":3820311304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6815","nonce":3820311304}]},"public_addr":"172.21.15.150:6811/3820311304","cluster_addr":"172.21.15.150:6813/3820311304","heartbeat_back_addr":"172.21.15.150:6817/3820311304","heartbeat_front_addr":"172.21.15.150:6815/3820311304","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6819","nonce":3881763618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6821","nonce":3881763618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6825","nonce":3881763618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6823","nonce":3881763618}]},"public_addr":"172.21.15.150:6819/3881763618","cluster_addr":"172.21.15.150:6821/3881763618","heartbeat_back_addr":"172.21.15.150:6825/3881763618","heartbeat_front_addr":"172.21.15.150:6823/3881763618","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6827","nonce":2956472788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6829","nonce":2956472788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6833","nonce":2956472788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6831","nonce":2956472788}]},"public_addr":"172.21.15.150:6827/2956472788","cluster_addr":"172.21.15.150:6829/2956472788","heartbeat_back_addr":"172.21.15.150:6833/2956472788","heartbeat_front_addr":"172.21.15.150:6831/2956472788","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:6801/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3200004174":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/2202195129":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3549400810":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2441406623":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T02:50:02.048 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-05T02:50:02.048 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: mds.? [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] up:boot 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:02.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:01 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2110456140' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: mds.? [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] up:boot 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2110456140' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: mds.? [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] up:boot 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:02.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:02.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:01 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2110456140' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:50:02.833 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:02.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:50:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[41818]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[29558]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:04.109 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:03 smithi196 ceph-mon[33169]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:03 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:04.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:04.806 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:50:04.807 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":69,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T02:49:49.199260+0000","last_up_change":"2024-04-05T02:48:00.713775+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T02:48:34.437290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T02:49:46.120483+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T02:49:46.414509+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6803","nonce":4116662696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6805","nonce":4116662696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6809","nonce":4116662696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4116662696},{"type":"v1","addr":"172.21.15.150:6807","nonce":4116662696}]},"public_addr":"172.21.15.150:6803/4116662696","cluster_addr":"172.21.15.150:6805/4116662696","heartbeat_back_addr":"172.21.15.150:6809/4116662696","heartbeat_front_addr":"172.21.15.150:6807/4116662696","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6811","nonce":3820311304}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6813","nonce":3820311304}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6817","nonce":3820311304}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3820311304},{"type":"v1","addr":"172.21.15.150:6815","nonce":3820311304}]},"public_addr":"172.21.15.150:6811/3820311304","cluster_addr":"172.21.15.150:6813/3820311304","heartbeat_back_addr":"172.21.15.150:6817/3820311304","heartbeat_front_addr":"172.21.15.150:6815/3820311304","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6819","nonce":3881763618}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6821","nonce":3881763618}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6825","nonce":3881763618}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":3881763618},{"type":"v1","addr":"172.21.15.150:6823","nonce":3881763618}]},"public_addr":"172.21.15.150:6819/3881763618","cluster_addr":"172.21.15.150:6821/3881763618","heartbeat_back_addr":"172.21.15.150:6825/3881763618","heartbeat_front_addr":"172.21.15.150:6823/3881763618","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6827","nonce":2956472788}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6829","nonce":2956472788}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6833","nonce":2956472788}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":2956472788},{"type":"v1","addr":"172.21.15.150:6831","nonce":2956472788}]},"public_addr":"172.21.15.150:6827/2956472788","cluster_addr":"172.21.15.150:6829/2956472788","heartbeat_back_addr":"172.21.15.150:6833/2956472788","heartbeat_front_addr":"172.21.15.150:6831/2956472788","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:6801/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3200004174":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/2202195129":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3549400810":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2441406623":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T02:50:05.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[29558]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:active 2024-04-05T02:50:05.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[29558]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:05.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:05.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:05.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/89361222' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:50:05.217 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[41818]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:active 2024-04-05T02:50:05.218 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:05.218 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:05.218 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:05.218 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:04 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/89361222' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:50:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:04 smithi196 ceph-mon[33169]: mds.? [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] up:active 2024-04-05T02:50:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:04 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:04 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:04 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:04 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/89361222' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T02:50:05.612 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-05T02:50:05.612 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-05T02:50:05.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-05T02:50:05.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-05T02:50:05.613 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-05T02:50:05.614 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-05T02:50:05.614 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-05T02:50:05.615 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-05T02:50:06.197 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:06.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:05 smithi150 ceph-mon[29558]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:06.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:05 smithi150 ceph-mon[29558]: mds.? [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] up:standby 2024-04-05T02:50:06.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:05 smithi150 ceph-mon[29558]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:06.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:05 smithi150 ceph-mon[41818]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:06.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:05 smithi150 ceph-mon[41818]: mds.? [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] up:standby 2024-04-05T02:50:06.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:05 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:05 smithi196 ceph-mon[33169]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:05 smithi196 ceph-mon[33169]: mds.? [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] up:standby 2024-04-05T02:50:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:05 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:50:06.623 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:06.623 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:06.623 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:06.995 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:06.997 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:08.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:08.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:08.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:08 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:08.145 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:08.146 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:08.147 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:08 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:10.294 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:10 smithi196 ceph-mon[33169]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:10.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:10 smithi150 ceph-mon[29558]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:10.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:10 smithi150 ceph-mon[41818]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:10.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:10] "GET /metrics HTTP/1.1" 200 239006 "" "Prometheus/2.43.0" 2024-04-05T02:50:10.561 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:12.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:11 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:12.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:11 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:12.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:11 smithi150 ceph-mon[29558]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:12.115 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:11 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:12.115 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:11 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:12.116 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:11 smithi150 ceph-mon[41818]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:11 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:11 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:11 smithi196 ceph-mon[33169]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:12.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:50:12 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:50:13.951 INFO:teuthology.orchestra.run.smithi150.stdout:201863462947 2024-04-05T02:50:13.951 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-05T02:50:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:13 smithi150 ceph-mon[29558]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:13 smithi150 ceph-mon[41818]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:13 smithi196 ceph-mon[33169]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:14.957 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:16.133 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:15 smithi150 ceph-mon[29558]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:16.133 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:15 smithi150 ceph-mon[41818]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:15 smithi196 ceph-mon[33169]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-04-05T02:50:17.038 INFO:teuthology.orchestra.run.smithi150.stdout:34359738432 2024-04-05T02:50:17.038 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-05T02:50:18.007 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:17 smithi150 ceph-mon[41818]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:18.007 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:17 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:18.007 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:17 smithi150 ceph-mon[29558]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:18.007 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:17 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:17 smithi196 ceph-mon[33169]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:17 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:19.499 INFO:teuthology.orchestra.run.smithi150.stdout:77309411383 2024-04-05T02:50:19.499 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-05T02:50:20.135 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:19 smithi150 ceph-mon[29558]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:20.147 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:19 smithi150 ceph-mon[41818]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:20.159 INFO:teuthology.orchestra.run.smithi150.stdout:167503724583 2024-04-05T02:50:20.159 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-05T02:50:20.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:19 smithi196 ceph-mon[33169]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:20.402 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:20] "GET /metrics HTTP/1.1" 200 239319 "" "Prometheus/2.43.0" 2024-04-05T02:50:21.953 INFO:teuthology.orchestra.run.smithi150.stdout:236223201311 2024-04-05T02:50:21.954 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-05T02:50:22.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:21 smithi150 ceph-mon[29558]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:22.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:21 smithi150 ceph-mon[41818]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:22.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:21 smithi196 ceph-mon[33169]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:22.407 INFO:teuthology.orchestra.run.smithi150.stdout:111669149746 2024-04-05T02:50:22.407 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-05T02:50:22.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:50:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:50:23.423 INFO:teuthology.orchestra.run.smithi150.stdout:133143986222 2024-04-05T02:50:23.423 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-05T02:50:23.934 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:23 smithi150 ceph-mon[41818]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:23.934 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:23 smithi150 ceph-mon[29558]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:23 smithi196 ceph-mon[33169]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:25.055 INFO:teuthology.orchestra.run.smithi150.stdout:55834574910 2024-04-05T02:50:25.055 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-05T02:50:25.129 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:25.227 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:26.190 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:25 smithi150 ceph-mon[29558]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:26.190 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:25 smithi150 ceph-mon[41818]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:25 smithi196 ceph-mon[33169]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:26.964 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:26.964 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:26.964 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:26.967 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:26.968 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:28.188 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:27 smithi196 ceph-mon[33169]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:28.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:27 smithi150 ceph-mon[29558]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:27 smithi150 ceph-mon[41818]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:29.462 INFO:teuthology.orchestra.run.smithi150.stdout:77309411386 2024-04-05T02:50:30.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:29 smithi150 ceph-mon[29558]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:30.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:29 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1570809420' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T02:50:30.217 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:29 smithi150 ceph-mon[41818]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:30.217 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1570809420' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T02:50:30.385 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:29 smithi196 ceph-mon[33169]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:30.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:29 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1570809420' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T02:50:30.476 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:30] "GET /metrics HTTP/1.1" 200 239319 "" "Prometheus/2.43.0" 2024-04-05T02:50:30.477 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-04-05T02:50:30.477 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:31 smithi150 ceph-mon[29558]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:31 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/811336401' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T02:50:32.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:31 smithi150 ceph-mon[41818]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:32.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:31 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/811336401' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T02:50:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:31 smithi196 ceph-mon[33169]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:31 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/811336401' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T02:50:32.876 INFO:teuthology.orchestra.run.smithi150.stdout:201863462951 2024-04-05T02:50:32.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:50:33.161 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.161 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.162 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-05T02:50:33.163 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:32 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:33.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-05T02:50:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]: dispatch 2024-04-05T02:50:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-04-05T02:50:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:32 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:33.627 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:33.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[29558]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:33.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-05T02:50:33.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-05T02:50:33.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-05T02:50:33.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[29558]: osdmap e70: 8 total, 8 up, 8 in 2024-04-05T02:50:33.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[41818]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:33.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-05T02:50:33.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-05T02:50:33.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-05T02:50:33.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:33 smithi150 ceph-mon[41818]: osdmap e70: 8 total, 8 up, 8 in 2024-04-05T02:50:34.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:33 smithi196 ceph-mon[33169]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:34.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:33 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-04-05T02:50:34.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:33 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 2]}]': finished 2024-04-05T02:50:34.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:33 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-04-05T02:50:34.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:33 smithi196 ceph-mon[33169]: osdmap e70: 8 total, 8 up, 8 in 2024-04-05T02:50:34.418 INFO:teuthology.orchestra.run.smithi150.stdout:133143986224 2024-04-05T02:50:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:34 smithi150 ceph-mon[29558]: osdmap e71: 8 total, 8 up, 8 in 2024-04-05T02:50:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:34 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1621928347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T02:50:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:34 smithi150 ceph-mon[41818]: osdmap e71: 8 total, 8 up, 8 in 2024-04-05T02:50:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:34 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1621928347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T02:50:35.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:34 smithi196 ceph-mon[33169]: osdmap e71: 8 total, 8 up, 8 in 2024-04-05T02:50:35.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:34 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1621928347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T02:50:36.064 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:35 smithi150 ceph-mon[29558]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:36.065 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:35 smithi150 ceph-mon[41818]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:36.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:35 smithi196 ceph-mon[33169]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:36.427 INFO:teuthology.orchestra.run.smithi150.stdout:111669149750 2024-04-05T02:50:37.197 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:36 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2413557181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T02:50:37.198 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:36 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2413557181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T02:50:37.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:36 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2413557181' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T02:50:37.645 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986222 got 133143986224 for osd.4 2024-04-05T02:50:37.645 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:38.147 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:37 smithi150 ceph-mon[29558]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:38.147 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:37 smithi150 ceph-mon[41818]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:38.170 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149750 for osd.3 2024-04-05T02:50:38.170 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:38.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:37 smithi196 ceph-mon[33169]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:38.913 INFO:teuthology.orchestra.run.smithi150.stdout:167503724587 2024-04-05T02:50:39.105 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:38 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3149961419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T02:50:39.105 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:38 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3149961419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T02:50:39.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:38 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3149961419' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T02:50:40.010 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462947 got 201863462951 for osd.6 2024-04-05T02:50:40.010 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:39 smithi196 ceph-mon[33169]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:39 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:39 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:39 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:39 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:39 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[29558]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[41818]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:40.212 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.213 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.213 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:40.213 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:40.213 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:39 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:40.430 INFO:teuthology.orchestra.run.smithi150.stdout:236223201315 2024-04-05T02:50:40.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:40] "GET /metrics HTTP/1.1" 200 239278 "" "Prometheus/2.43.0" 2024-04-05T02:50:41.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:40 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3170161094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T02:50:41.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:40 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3170161094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T02:50:41.361 INFO:teuthology.orchestra.run.smithi150.stdout:34359738437 2024-04-05T02:50:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:40 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3170161094' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T02:50:41.787 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201315 for osd.7 2024-04-05T02:50:41.787 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:42.207 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:41 smithi150 ceph-mon[41818]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:42.208 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:41 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1118845022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T02:50:42.208 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:41 smithi150 ceph-mon[29558]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:42.208 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:41 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1118845022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T02:50:42.232 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738432 got 34359738437 for osd.0 2024-04-05T02:50:42.233 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:41 smithi196 ceph-mon[33169]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:42.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:41 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1118845022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T02:50:42.701 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724583 got 167503724587 for osd.5 2024-04-05T02:50:42.701 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:42.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:50:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:50:43.344 INFO:teuthology.orchestra.run.smithi150.stdout:55834574913 2024-04-05T02:50:44.029 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574910 got 55834574913 for osd.1 2024-04-05T02:50:44.030 DEBUG:teuthology.parallel:result is None 2024-04-05T02:50:44.030 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-05T02:50:44.030 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T02:50:44.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:43 smithi150 ceph-mon[29558]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:44.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:43 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/195961873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T02:50:44.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:43 smithi150 ceph-mon[41818]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:44.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:43 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/195961873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T02:50:44.290 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:43 smithi196 ceph-mon[33169]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:43 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/195961873' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T02:50:46.189 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:50:46.190 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:45 smithi150 ceph-mon[29558]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:46.190 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:45 smithi150 ceph-mon[41818]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:46.195 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-04-05T02:50:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:45 smithi196 ceph-mon[33169]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:46.945 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":80,"stamp":"2024-04-05T02:50:44.851854+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386352,"kb_used_data":7480,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574864,"statfs":{"total":767960285184,"available":765516660736,"internally_reserved":0,"allocated":7659520,"data_stored":4308005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.923164"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.420910+0000","last_change":"2024-04-05T02:49:47.206413+0000","last_active":"2024-04-05T02:49:50.420910+0000","last_peered":"2024-04-05T02:49:50.420910+0000","last_clean":"2024-04-05T02:49:50.420910+0000","last_became_active":"2024-04-05T02:49:47.206099+0000","last_became_peered":"2024-04-05T02:49:47.206099+0000","last_unstale":"2024-04-05T02:49:50.420910+0000","last_undegraded":"2024-04-05T02:49:50.420910+0000","last_fullsized":"2024-04-05T02:49:50.420910+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:36:28.371339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.912568+0000","last_change":"2024-04-05T02:49:47.198309+0000","last_active":"2024-04-05T02:49:58.912568+0000","last_peered":"2024-04-05T02:49:58.912568+0000","last_clean":"2024-04-05T02:49:58.912568+0000","last_became_active":"2024-04-05T02:49:47.197378+0000","last_became_peered":"2024-04-05T02:49:47.197378+0000","last_unstale":"2024-04-05T02:49:58.912568+0000","last_undegraded":"2024-04-05T02:49:58.912568+0000","last_fullsized":"2024-04-05T02:49:58.912568+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:44:03.975757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928554+0000","last_change":"2024-04-05T02:49:48.245928+0000","last_active":"2024-04-05T02:50:32.928554+0000","last_peered":"2024-04-05T02:50:32.928554+0000","last_clean":"2024-04-05T02:50:32.928554+0000","last_became_active":"2024-04-05T02:49:48.245582+0000","last_became_peered":"2024-04-05T02:49:48.245582+0000","last_unstale":"2024-04-05T02:50:32.928554+0000","last_undegraded":"2024-04-05T02:50:32.928554+0000","last_fullsized":"2024-04-05T02:50:32.928554+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:19:54.560500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929051+0000","last_change":"2024-04-05T02:49:47.194809+0000","last_active":"2024-04-05T02:50:32.929051+0000","last_peered":"2024-04-05T02:50:32.929051+0000","last_clean":"2024-04-05T02:50:32.929051+0000","last_became_active":"2024-04-05T02:49:47.194535+0000","last_became_peered":"2024-04-05T02:49:47.194535+0000","last_unstale":"2024-04-05T02:50:32.929051+0000","last_undegraded":"2024-04-05T02:50:32.929051+0000","last_fullsized":"2024-04-05T02:50:32.929051+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:26:50.573010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421291+0000","last_change":"2024-04-05T02:49:47.247249+0000","last_active":"2024-04-05T02:49:50.421291+0000","last_peered":"2024-04-05T02:49:50.421291+0000","last_clean":"2024-04-05T02:49:50.421291+0000","last_became_active":"2024-04-05T02:49:47.246981+0000","last_became_peered":"2024-04-05T02:49:47.246981+0000","last_unstale":"2024-04-05T02:49:50.421291+0000","last_undegraded":"2024-04-05T02:49:50.421291+0000","last_fullsized":"2024-04-05T02:49:50.421291+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:57:52.416336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930443+0000","last_change":"2024-04-05T02:49:47.188962+0000","last_active":"2024-04-05T02:50:32.930443+0000","last_peered":"2024-04-05T02:50:32.930443+0000","last_clean":"2024-04-05T02:50:32.930443+0000","last_became_active":"2024-04-05T02:49:47.188681+0000","last_became_peered":"2024-04-05T02:49:47.188681+0000","last_unstale":"2024-04-05T02:50:32.930443+0000","last_undegraded":"2024-04-05T02:50:32.930443+0000","last_fullsized":"2024-04-05T02:50:32.930443+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:59:25.089268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421985+0000","last_change":"2024-04-05T02:49:47.248544+0000","last_active":"2024-04-05T02:49:50.421985+0000","last_peered":"2024-04-05T02:49:50.421985+0000","last_clean":"2024-04-05T02:49:50.421985+0000","last_became_active":"2024-04-05T02:49:47.248214+0000","last_became_peered":"2024-04-05T02:49:47.248214+0000","last_unstale":"2024-04-05T02:49:50.421985+0000","last_undegraded":"2024-04-05T02:49:50.421985+0000","last_fullsized":"2024-04-05T02:49:50.421985+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:16:55.500018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930286+0000","last_change":"2024-04-05T02:49:47.191401+0000","last_active":"2024-04-05T02:50:32.930286+0000","last_peered":"2024-04-05T02:50:32.930286+0000","last_clean":"2024-04-05T02:50:32.930286+0000","last_became_active":"2024-04-05T02:49:47.190758+0000","last_became_peered":"2024-04-05T02:49:47.190758+0000","last_unstale":"2024-04-05T02:50:32.930286+0000","last_undegraded":"2024-04-05T02:50:32.930286+0000","last_fullsized":"2024-04-05T02:50:32.930286+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:24.309425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-05T02:50:33.949153+0000","last_change":"2024-04-05T02:50:33.949153+0000","last_active":"2024-04-05T02:50:33.949153+0000","last_peered":"2024-04-05T02:50:33.949153+0000","last_clean":"2024-04-05T02:50:33.949153+0000","last_became_active":"2024-04-05T02:50:33.948818+0000","last_became_peered":"2024-04-05T02:50:33.948818+0000","last_unstale":"2024-04-05T02:50:33.949153+0000","last_undegraded":"2024-04-05T02:50:33.949153+0000","last_fullsized":"2024-04-05T02:50:33.949153+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:53.952889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.907919+0000","last_change":"2024-04-05T02:49:47.195429+0000","last_active":"2024-04-05T02:49:58.907919+0000","last_peered":"2024-04-05T02:49:58.907919+0000","last_clean":"2024-04-05T02:49:58.907919+0000","last_became_active":"2024-04-05T02:49:47.195063+0000","last_became_peered":"2024-04-05T02:49:47.195063+0000","last_unstale":"2024-04-05T02:49:58.907919+0000","last_undegraded":"2024-04-05T02:49:58.907919+0000","last_fullsized":"2024-04-05T02:49:58.907919+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:23:16.328321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421875+0000","last_change":"2024-04-05T02:49:47.206433+0000","last_active":"2024-04-05T02:49:50.421875+0000","last_peered":"2024-04-05T02:49:50.421875+0000","last_clean":"2024-04-05T02:49:50.421875+0000","last_became_active":"2024-04-05T02:49:47.206035+0000","last_became_peered":"2024-04-05T02:49:47.206035+0000","last_unstale":"2024-04-05T02:49:50.421875+0000","last_undegraded":"2024-04-05T02:49:50.421875+0000","last_fullsized":"2024-04-05T02:49:50.421875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:08:55.149404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:50:03.303344+0000","last_change":"2024-04-05T02:49:47.200921+0000","last_active":"2024-04-05T02:50:03.303344+0000","last_peered":"2024-04-05T02:50:03.303344+0000","last_clean":"2024-04-05T02:50:03.303344+0000","last_became_active":"2024-04-05T02:49:47.200539+0000","last_became_peered":"2024-04-05T02:49:47.200539+0000","last_unstale":"2024-04-05T02:50:03.303344+0000","last_undegraded":"2024-04-05T02:50:03.303344+0000","last_fullsized":"2024-04-05T02:50:03.303344+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:43:09.109344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933266+0000","last_change":"2024-04-05T02:48:35.842380+0000","last_active":"2024-04-05T02:50:32.933266+0000","last_peered":"2024-04-05T02:50:32.933266+0000","last_clean":"2024-04-05T02:50:32.933266+0000","last_became_active":"2024-04-05T02:48:35.842036+0000","last_became_peered":"2024-04-05T02:48:35.842036+0000","last_unstale":"2024-04-05T02:50:32.933266+0000","last_undegraded":"2024-04-05T02:50:32.933266+0000","last_fullsized":"2024-04-05T02:50:32.933266+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:50.319270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928433+0000","last_change":"2024-04-05T02:49:48.222222+0000","last_active":"2024-04-05T02:50:32.928433+0000","last_peered":"2024-04-05T02:50:32.928433+0000","last_clean":"2024-04-05T02:50:32.928433+0000","last_became_active":"2024-04-05T02:49:48.221172+0000","last_became_peered":"2024-04-05T02:49:48.221172+0000","last_unstale":"2024-04-05T02:50:32.928433+0000","last_undegraded":"2024-04-05T02:50:32.928433+0000","last_fullsized":"2024-04-05T02:50:32.928433+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:17:29.162596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.932073+0000","last_change":"2024-04-05T02:48:36.012430+0000","last_active":"2024-04-05T02:50:32.932073+0000","last_peered":"2024-04-05T02:50:32.932073+0000","last_clean":"2024-04-05T02:50:32.932073+0000","last_became_active":"2024-04-05T02:48:36.012069+0000","last_became_peered":"2024-04-05T02:48:36.012069+0000","last_unstale":"2024-04-05T02:50:32.932073+0000","last_undegraded":"2024-04-05T02:50:32.932073+0000","last_fullsized":"2024-04-05T02:50:32.932073+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T02:59:27.856396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929113+0000","last_change":"2024-04-05T02:49:47.199782+0000","last_active":"2024-04-05T02:50:32.929113+0000","last_peered":"2024-04-05T02:50:32.929113+0000","last_clean":"2024-04-05T02:50:32.929113+0000","last_became_active":"2024-04-05T02:49:47.198795+0000","last_became_peered":"2024-04-05T02:49:47.198795+0000","last_unstale":"2024-04-05T02:50:32.929113+0000","last_undegraded":"2024-04-05T02:50:32.929113+0000","last_fullsized":"2024-04-05T02:50:32.929113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:41:38.102537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203302+0000","last_change":"2024-04-05T02:49:48.222221+0000","last_active":"2024-04-05T02:49:49.203302+0000","last_peered":"2024-04-05T02:49:49.203302+0000","last_clean":"2024-04-05T02:49:49.203302+0000","last_became_active":"2024-04-05T02:49:48.221228+0000","last_became_peered":"2024-04-05T02:49:48.221228+0000","last_unstale":"2024-04-05T02:49:49.203302+0000","last_undegraded":"2024-04-05T02:49:49.203302+0000","last_fullsized":"2024-04-05T02:49:49.203302+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:17.668761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931785+0000","last_change":"2024-04-05T02:49:47.197146+0000","last_active":"2024-04-05T02:50:32.931785+0000","last_peered":"2024-04-05T02:50:32.931785+0000","last_clean":"2024-04-05T02:50:32.931785+0000","last_became_active":"2024-04-05T02:49:47.194951+0000","last_became_peered":"2024-04-05T02:49:47.194951+0000","last_unstale":"2024-04-05T02:50:32.931785+0000","last_undegraded":"2024-04-05T02:50:32.931785+0000","last_fullsized":"2024-04-05T02:50:32.931785+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:19:07.376060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931221+0000","last_change":"2024-04-05T02:48:36.033701+0000","last_active":"2024-04-05T02:50:32.931221+0000","last_peered":"2024-04-05T02:50:32.931221+0000","last_clean":"2024-04-05T02:50:32.931221+0000","last_became_active":"2024-04-05T02:48:36.033383+0000","last_became_peered":"2024-04-05T02:48:36.033383+0000","last_unstale":"2024-04-05T02:50:32.931221+0000","last_undegraded":"2024-04-05T02:50:32.931221+0000","last_fullsized":"2024-04-05T02:50:32.931221+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:57:58.141349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203246+0000","last_change":"2024-04-05T02:49:48.246020+0000","last_active":"2024-04-05T02:49:49.203246+0000","last_peered":"2024-04-05T02:49:49.203246+0000","last_clean":"2024-04-05T02:49:49.203246+0000","last_became_active":"2024-04-05T02:49:48.245722+0000","last_became_peered":"2024-04-05T02:49:48.245722+0000","last_unstale":"2024-04-05T02:49:49.203246+0000","last_undegraded":"2024-04-05T02:49:49.203246+0000","last_fullsized":"2024-04-05T02:49:49.203246+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:20:20.342453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.202995+0000","last_change":"2024-04-05T02:49:47.192884+0000","last_active":"2024-04-05T02:49:49.202995+0000","last_peered":"2024-04-05T02:49:49.202995+0000","last_clean":"2024-04-05T02:49:49.202995+0000","last_became_active":"2024-04-05T02:49:47.192527+0000","last_became_peered":"2024-04-05T02:49:47.192527+0000","last_unstale":"2024-04-05T02:49:49.202995+0000","last_undegraded":"2024-04-05T02:49:49.202995+0000","last_fullsized":"2024-04-05T02:49:49.202995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:58:54.090853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931093+0000","last_change":"2024-04-05T02:48:36.033673+0000","last_active":"2024-04-05T02:50:32.931093+0000","last_peered":"2024-04-05T02:50:32.931093+0000","last_clean":"2024-04-05T02:50:32.931093+0000","last_became_active":"2024-04-05T02:48:36.033304+0000","last_became_peered":"2024-04-05T02:48:36.033304+0000","last_unstale":"2024-04-05T02:50:32.931093+0000","last_undegraded":"2024-04-05T02:50:32.931093+0000","last_fullsized":"2024-04-05T02:50:32.931093+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:26.516492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933230+0000","last_change":"2024-04-05T02:49:47.246904+0000","last_active":"2024-04-05T02:50:32.933230+0000","last_peered":"2024-04-05T02:50:32.933230+0000","last_clean":"2024-04-05T02:50:32.933230+0000","last_became_active":"2024-04-05T02:49:47.246606+0000","last_became_peered":"2024-04-05T02:49:47.246606+0000","last_unstale":"2024-04-05T02:50:32.933230+0000","last_undegraded":"2024-04-05T02:50:32.933230+0000","last_fullsized":"2024-04-05T02:50:32.933230+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:18:33.167051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928805+0000","last_change":"2024-04-05T02:49:47.196017+0000","last_active":"2024-04-05T02:50:32.928805+0000","last_peered":"2024-04-05T02:50:32.928805+0000","last_clean":"2024-04-05T02:50:32.928805+0000","last_became_active":"2024-04-05T02:49:47.195723+0000","last_became_peered":"2024-04-05T02:49:47.195723+0000","last_unstale":"2024-04-05T02:50:32.928805+0000","last_undegraded":"2024-04-05T02:50:32.928805+0000","last_fullsized":"2024-04-05T02:50:32.928805+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:35:20.900943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":273,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931029+0000","last_change":"2024-04-05T02:48:04.212692+0000","last_active":"2024-04-05T02:50:32.931029+0000","last_peered":"2024-04-05T02:50:32.931029+0000","last_clean":"2024-04-05T02:50:32.931029+0000","last_became_active":"2024-04-05T02:48:04.212212+0000","last_became_peered":"2024-04-05T02:48:04.212212+0000","last_unstale":"2024-04-05T02:50:32.931029+0000","last_undegraded":"2024-04-05T02:50:32.931029+0000","last_fullsized":"2024-04-05T02:50:32.931029+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_clean_scrub_stamp":"2024-04-05T02:45:59.919732+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:33:13.395158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930145+0000","last_change":"2024-04-05T02:48:35.842260+0000","last_active":"2024-04-05T02:50:32.930145+0000","last_peered":"2024-04-05T02:50:32.930145+0000","last_clean":"2024-04-05T02:50:32.930145+0000","last_became_active":"2024-04-05T02:48:35.841968+0000","last_became_peered":"2024-04-05T02:48:35.841968+0000","last_unstale":"2024-04-05T02:50:32.930145+0000","last_undegraded":"2024-04-05T02:50:32.930145+0000","last_fullsized":"2024-04-05T02:50:32.930145+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:37:28.928023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931588+0000","last_change":"2024-04-05T02:49:48.209094+0000","last_active":"2024-04-05T02:50:32.931588+0000","last_peered":"2024-04-05T02:50:32.931588+0000","last_clean":"2024-04-05T02:50:32.931588+0000","last_became_active":"2024-04-05T02:49:48.208816+0000","last_became_peered":"2024-04-05T02:49:48.208816+0000","last_unstale":"2024-04-05T02:50:32.931588+0000","last_undegraded":"2024-04-05T02:50:32.931588+0000","last_fullsized":"2024-04-05T02:50:32.931588+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:49:54.730675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931401+0000","last_change":"2024-04-05T02:49:47.196981+0000","last_active":"2024-04-05T02:50:32.931401+0000","last_peered":"2024-04-05T02:50:32.931401+0000","last_clean":"2024-04-05T02:50:32.931401+0000","last_became_active":"2024-04-05T02:49:47.196253+0000","last_became_peered":"2024-04-05T02:49:47.196253+0000","last_unstale":"2024-04-05T02:50:32.931401+0000","last_undegraded":"2024-04-05T02:50:32.931401+0000","last_fullsized":"2024-04-05T02:50:32.931401+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:46:10.152538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930917+0000","last_change":"2024-04-05T02:48:36.033794+0000","last_active":"2024-04-05T02:50:32.930917+0000","last_peered":"2024-04-05T02:50:32.930917+0000","last_clean":"2024-04-05T02:50:32.930917+0000","last_became_active":"2024-04-05T02:48:36.033472+0000","last_became_peered":"2024-04-05T02:48:36.033472+0000","last_unstale":"2024-04-05T02:50:32.930917+0000","last_undegraded":"2024-04-05T02:50:32.930917+0000","last_fullsized":"2024-04-05T02:50:32.930917+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:39:42.977557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931715+0000","last_change":"2024-04-05T02:49:48.246485+0000","last_active":"2024-04-05T02:50:32.931715+0000","last_peered":"2024-04-05T02:50:32.931715+0000","last_clean":"2024-04-05T02:50:32.931715+0000","last_became_active":"2024-04-05T02:49:48.244873+0000","last_became_peered":"2024-04-05T02:49:48.244873+0000","last_unstale":"2024-04-05T02:50:32.931715+0000","last_undegraded":"2024-04-05T02:50:32.931715+0000","last_fullsized":"2024-04-05T02:50:32.931715+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:22.827371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.422071+0000","last_change":"2024-04-05T02:49:47.192710+0000","last_active":"2024-04-05T02:49:50.422071+0000","last_peered":"2024-04-05T02:49:50.422071+0000","last_clean":"2024-04-05T02:49:50.422071+0000","last_became_active":"2024-04-05T02:49:47.192355+0000","last_became_peered":"2024-04-05T02:49:47.192355+0000","last_unstale":"2024-04-05T02:49:50.422071+0000","last_undegraded":"2024-04-05T02:49:50.422071+0000","last_fullsized":"2024-04-05T02:49:50.422071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:39:34.153715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929998+0000","last_change":"2024-04-05T02:48:35.840743+0000","last_active":"2024-04-05T02:50:32.929998+0000","last_peered":"2024-04-05T02:50:32.929998+0000","last_clean":"2024-04-05T02:50:32.929998+0000","last_became_active":"2024-04-05T02:48:35.840424+0000","last_became_peered":"2024-04-05T02:48:35.840424+0000","last_unstale":"2024-04-05T02:50:32.929998+0000","last_undegraded":"2024-04-05T02:50:32.929998+0000","last_fullsized":"2024-04-05T02:50:32.929998+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:04:16.216370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931634+0000","last_change":"2024-04-05T02:49:48.246029+0000","last_active":"2024-04-05T02:50:32.931634+0000","last_peered":"2024-04-05T02:50:32.931634+0000","last_clean":"2024-04-05T02:50:32.931634+0000","last_became_active":"2024-04-05T02:49:48.245716+0000","last_became_peered":"2024-04-05T02:49:48.245716+0000","last_unstale":"2024-04-05T02:50:32.931634+0000","last_undegraded":"2024-04-05T02:50:32.931634+0000","last_fullsized":"2024-04-05T02:50:32.931634+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:07:34.775055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930416+0000","last_change":"2024-04-05T02:49:47.190160+0000","last_active":"2024-04-05T02:50:32.930416+0000","last_peered":"2024-04-05T02:50:32.930416+0000","last_clean":"2024-04-05T02:50:32.930416+0000","last_became_active":"2024-04-05T02:49:47.189855+0000","last_became_peered":"2024-04-05T02:49:47.189855+0000","last_unstale":"2024-04-05T02:50:32.930416+0000","last_undegraded":"2024-04-05T02:50:32.930416+0000","last_fullsized":"2024-04-05T02:50:32.930416+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:48:04.061243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931642+0000","last_change":"2024-04-05T02:48:36.034060+0000","last_active":"2024-04-05T02:50:32.931642+0000","last_peered":"2024-04-05T02:50:32.931642+0000","last_clean":"2024-04-05T02:50:32.931642+0000","last_became_active":"2024-04-05T02:48:36.033747+0000","last_became_peered":"2024-04-05T02:48:36.033747+0000","last_unstale":"2024-04-05T02:50:32.931642+0000","last_undegraded":"2024-04-05T02:50:32.931642+0000","last_fullsized":"2024-04-05T02:50:32.931642+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:16:53.864447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931671+0000","last_change":"2024-04-05T02:49:48.245229+0000","last_active":"2024-04-05T02:50:32.931671+0000","last_peered":"2024-04-05T02:50:32.931671+0000","last_clean":"2024-04-05T02:50:32.931671+0000","last_became_active":"2024-04-05T02:49:48.244920+0000","last_became_peered":"2024-04-05T02:49:48.244920+0000","last_unstale":"2024-04-05T02:50:32.931671+0000","last_undegraded":"2024-04-05T02:50:32.931671+0000","last_fullsized":"2024-04-05T02:50:32.931671+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:22:28.965318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928457+0000","last_change":"2024-04-05T02:49:47.196272+0000","last_active":"2024-04-05T02:50:32.928457+0000","last_peered":"2024-04-05T02:50:32.928457+0000","last_clean":"2024-04-05T02:50:32.928457+0000","last_became_active":"2024-04-05T02:49:47.195949+0000","last_became_peered":"2024-04-05T02:49:47.195949+0000","last_unstale":"2024-04-05T02:50:32.928457+0000","last_undegraded":"2024-04-05T02:50:32.928457+0000","last_fullsized":"2024-04-05T02:50:32.928457+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:14:30.814375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421240+0000","last_change":"2024-04-05T02:49:47.206746+0000","last_active":"2024-04-05T02:49:50.421240+0000","last_peered":"2024-04-05T02:49:50.421240+0000","last_clean":"2024-04-05T02:49:50.421240+0000","last_became_active":"2024-04-05T02:49:47.206386+0000","last_became_peered":"2024-04-05T02:49:47.206386+0000","last_unstale":"2024-04-05T02:49:50.421240+0000","last_undegraded":"2024-04-05T02:49:50.421240+0000","last_fullsized":"2024-04-05T02:49:50.421240+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T02:50:27.026708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421899+0000","last_change":"2024-04-05T02:49:47.187686+0000","last_active":"2024-04-05T02:49:50.421899+0000","last_peered":"2024-04-05T02:49:50.421899+0000","last_clean":"2024-04-05T02:49:50.421899+0000","last_became_active":"2024-04-05T02:49:47.187375+0000","last_became_peered":"2024-04-05T02:49:47.187375+0000","last_unstale":"2024-04-05T02:49:50.421899+0000","last_undegraded":"2024-04-05T02:49:50.421899+0000","last_fullsized":"2024-04-05T02:49:50.421899+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:21:13.280128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421765+0000","last_change":"2024-04-05T02:49:47.210374+0000","last_active":"2024-04-05T02:49:50.421765+0000","last_peered":"2024-04-05T02:49:50.421765+0000","last_clean":"2024-04-05T02:49:50.421765+0000","last_became_active":"2024-04-05T02:49:47.210128+0000","last_became_peered":"2024-04-05T02:49:47.210128+0000","last_unstale":"2024-04-05T02:49:50.421765+0000","last_undegraded":"2024-04-05T02:49:50.421765+0000","last_fullsized":"2024-04-05T02:49:50.421765+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:02:46.410428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933879+0000","last_change":"2024-04-05T02:49:47.189496+0000","last_active":"2024-04-05T02:50:32.933879+0000","last_peered":"2024-04-05T02:50:32.933879+0000","last_clean":"2024-04-05T02:50:32.933879+0000","last_became_active":"2024-04-05T02:49:47.189216+0000","last_became_peered":"2024-04-05T02:49:47.189216+0000","last_unstale":"2024-04-05T02:50:32.933879+0000","last_undegraded":"2024-04-05T02:50:32.933879+0000","last_fullsized":"2024-04-05T02:50:32.933879+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:03.749329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421077+0000","last_change":"2024-04-05T02:49:47.248235+0000","last_active":"2024-04-05T02:49:50.421077+0000","last_peered":"2024-04-05T02:49:50.421077+0000","last_clean":"2024-04-05T02:49:50.421077+0000","last_became_active":"2024-04-05T02:49:47.247680+0000","last_became_peered":"2024-04-05T02:49:47.247680+0000","last_unstale":"2024-04-05T02:49:50.421077+0000","last_undegraded":"2024-04-05T02:49:50.421077+0000","last_fullsized":"2024-04-05T02:49:50.421077+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:36.464869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928812+0000","last_change":"2024-04-05T02:49:47.197456+0000","last_active":"2024-04-05T02:50:32.928812+0000","last_peered":"2024-04-05T02:50:32.928812+0000","last_clean":"2024-04-05T02:50:32.928812+0000","last_became_active":"2024-04-05T02:49:47.197145+0000","last_became_peered":"2024-04-05T02:49:47.197145+0000","last_unstale":"2024-04-05T02:50:32.928812+0000","last_undegraded":"2024-04-05T02:50:32.928812+0000","last_fullsized":"2024-04-05T02:50:32.928812+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:06:00.892577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.202988+0000","last_change":"2024-04-05T02:49:48.247884+0000","last_active":"2024-04-05T02:49:49.202988+0000","last_peered":"2024-04-05T02:49:49.202988+0000","last_clean":"2024-04-05T02:49:49.202988+0000","last_became_active":"2024-04-05T02:49:48.247548+0000","last_became_peered":"2024-04-05T02:49:48.247548+0000","last_unstale":"2024-04-05T02:49:49.202988+0000","last_undegraded":"2024-04-05T02:49:49.202988+0000","last_fullsized":"2024-04-05T02:49:49.202988+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:21.723925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930273+0000","last_change":"2024-04-05T02:49:47.190298+0000","last_active":"2024-04-05T02:50:32.930273+0000","last_peered":"2024-04-05T02:50:32.930273+0000","last_clean":"2024-04-05T02:50:32.930273+0000","last_became_active":"2024-04-05T02:49:47.189872+0000","last_became_peered":"2024-04-05T02:49:47.189872+0000","last_unstale":"2024-04-05T02:50:32.930273+0000","last_undegraded":"2024-04-05T02:50:32.930273+0000","last_fullsized":"2024-04-05T02:50:32.930273+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:47:45.371257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-05T02:50:33.934126+0000","last_change":"2024-04-05T02:50:33.934126+0000","last_active":"2024-04-05T02:50:33.934126+0000","last_peered":"2024-04-05T02:50:33.934126+0000","last_clean":"2024-04-05T02:50:33.934126+0000","last_became_active":"2024-04-05T02:50:33.933725+0000","last_became_peered":"2024-04-05T02:50:33.933725+0000","last_unstale":"2024-04-05T02:50:33.934126+0000","last_undegraded":"2024-04-05T02:50:33.934126+0000","last_fullsized":"2024-04-05T02:50:33.934126+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:47:08.848659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931292+0000","last_change":"2024-04-05T02:49:47.188385+0000","last_active":"2024-04-05T02:50:32.931292+0000","last_peered":"2024-04-05T02:50:32.931292+0000","last_clean":"2024-04-05T02:50:32.931292+0000","last_became_active":"2024-04-05T02:49:47.188108+0000","last_became_peered":"2024-04-05T02:49:47.188108+0000","last_unstale":"2024-04-05T02:50:32.931292+0000","last_undegraded":"2024-04-05T02:50:32.931292+0000","last_fullsized":"2024-04-05T02:50:32.931292+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:01.691480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421736+0000","last_change":"2024-04-05T02:49:47.246658+0000","last_active":"2024-04-05T02:49:50.421736+0000","last_peered":"2024-04-05T02:49:50.421736+0000","last_clean":"2024-04-05T02:49:50.421736+0000","last_became_active":"2024-04-05T02:49:47.246374+0000","last_became_peered":"2024-04-05T02:49:47.246374+0000","last_unstale":"2024-04-05T02:49:50.421736+0000","last_undegraded":"2024-04-05T02:49:50.421736+0000","last_fullsized":"2024-04-05T02:49:50.421736+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:19:09.248707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931368+0000","last_change":"2024-04-05T02:49:47.192997+0000","last_active":"2024-04-05T02:50:32.931368+0000","last_peered":"2024-04-05T02:50:32.931368+0000","last_clean":"2024-04-05T02:50:32.931368+0000","last_became_active":"2024-04-05T02:49:47.192662+0000","last_became_peered":"2024-04-05T02:49:47.192662+0000","last_unstale":"2024-04-05T02:50:32.931368+0000","last_undegraded":"2024-04-05T02:50:32.931368+0000","last_fullsized":"2024-04-05T02:50:32.931368+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:42:21.816507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929953+0000","last_change":"2024-04-05T02:49:47.246329+0000","last_active":"2024-04-05T02:50:32.929953+0000","last_peered":"2024-04-05T02:50:32.929953+0000","last_clean":"2024-04-05T02:50:32.929953+0000","last_became_active":"2024-04-05T02:49:47.245961+0000","last_became_peered":"2024-04-05T02:49:47.245961+0000","last_unstale":"2024-04-05T02:50:32.929953+0000","last_undegraded":"2024-04-05T02:50:32.929953+0000","last_fullsized":"2024-04-05T02:50:32.929953+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:45:18.083543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931468+0000","last_change":"2024-04-05T02:49:47.192903+0000","last_active":"2024-04-05T02:50:32.931468+0000","last_peered":"2024-04-05T02:50:32.931468+0000","last_clean":"2024-04-05T02:50:32.931468+0000","last_became_active":"2024-04-05T02:49:47.192626+0000","last_became_peered":"2024-04-05T02:49:47.192626+0000","last_unstale":"2024-04-05T02:50:32.931468+0000","last_undegraded":"2024-04-05T02:50:32.931468+0000","last_fullsized":"2024-04-05T02:50:32.931468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:48:07.433594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929024+0000","last_change":"2024-04-05T02:49:48.222667+0000","last_active":"2024-04-05T02:50:32.929024+0000","last_peered":"2024-04-05T02:50:32.929024+0000","last_clean":"2024-04-05T02:50:32.929024+0000","last_became_active":"2024-04-05T02:49:48.221235+0000","last_became_peered":"2024-04-05T02:49:48.221235+0000","last_unstale":"2024-04-05T02:50:32.929024+0000","last_undegraded":"2024-04-05T02:50:32.929024+0000","last_fullsized":"2024-04-05T02:50:32.929024+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:11:26.797834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.932959+0000","last_change":"2024-04-05T02:49:47.188396+0000","last_active":"2024-04-05T02:50:32.932959+0000","last_peered":"2024-04-05T02:50:32.932959+0000","last_clean":"2024-04-05T02:50:32.932959+0000","last_became_active":"2024-04-05T02:49:47.188129+0000","last_became_peered":"2024-04-05T02:49:47.188129+0000","last_unstale":"2024-04-05T02:50:32.932959+0000","last_undegraded":"2024-04-05T02:50:32.932959+0000","last_fullsized":"2024-04-05T02:50:32.932959+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:07:47.941423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203056+0000","last_change":"2024-04-05T02:49:48.221773+0000","last_active":"2024-04-05T02:49:49.203056+0000","last_peered":"2024-04-05T02:49:49.203056+0000","last_clean":"2024-04-05T02:49:49.203056+0000","last_became_active":"2024-04-05T02:49:48.221402+0000","last_became_peered":"2024-04-05T02:49:48.221402+0000","last_unstale":"2024-04-05T02:49:49.203056+0000","last_undegraded":"2024-04-05T02:49:49.203056+0000","last_fullsized":"2024-04-05T02:49:49.203056+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:47.436908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930818+0000","last_change":"2024-04-05T02:49:47.190900+0000","last_active":"2024-04-05T02:50:32.930818+0000","last_peered":"2024-04-05T02:50:32.930818+0000","last_clean":"2024-04-05T02:50:32.930818+0000","last_became_active":"2024-04-05T02:49:47.190633+0000","last_became_peered":"2024-04-05T02:49:47.190633+0000","last_unstale":"2024-04-05T02:50:32.930818+0000","last_undegraded":"2024-04-05T02:50:32.930818+0000","last_fullsized":"2024-04-05T02:50:32.930818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:59:46.877238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930335+0000","last_change":"2024-04-05T02:49:47.248198+0000","last_active":"2024-04-05T02:50:32.930335+0000","last_peered":"2024-04-05T02:50:32.930335+0000","last_clean":"2024-04-05T02:50:32.930335+0000","last_became_active":"2024-04-05T02:49:47.247847+0000","last_became_peered":"2024-04-05T02:49:47.247847+0000","last_unstale":"2024-04-05T02:50:32.930335+0000","last_undegraded":"2024-04-05T02:50:32.930335+0000","last_fullsized":"2024-04-05T02:50:32.930335+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:39:30.162920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.202914+0000","last_change":"2024-04-05T02:49:47.191362+0000","last_active":"2024-04-05T02:49:49.202914+0000","last_peered":"2024-04-05T02:49:49.202914+0000","last_clean":"2024-04-05T02:49:49.202914+0000","last_became_active":"2024-04-05T02:49:47.191040+0000","last_became_peered":"2024-04-05T02:49:47.191040+0000","last_unstale":"2024-04-05T02:49:49.202914+0000","last_undegraded":"2024-04-05T02:49:49.202914+0000","last_fullsized":"2024-04-05T02:49:49.202914+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:40:39.022369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-05T02:50:33.934345+0000","last_change":"2024-04-05T02:50:33.934345+0000","last_active":"2024-04-05T02:50:33.934345+0000","last_peered":"2024-04-05T02:50:33.934345+0000","last_clean":"2024-04-05T02:50:33.934345+0000","last_became_active":"2024-04-05T02:50:33.933968+0000","last_became_peered":"2024-04-05T02:50:33.933968+0000","last_unstale":"2024-04-05T02:50:33.934345+0000","last_undegraded":"2024-04-05T02:50:33.934345+0000","last_fullsized":"2024-04-05T02:50:33.934345+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:58:29.769992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931029+0000","last_change":"2024-04-05T02:49:47.191189+0000","last_active":"2024-04-05T02:50:32.931029+0000","last_peered":"2024-04-05T02:50:32.931029+0000","last_clean":"2024-04-05T02:50:32.931029+0000","last_became_active":"2024-04-05T02:49:47.190925+0000","last_became_peered":"2024-04-05T02:49:47.190925+0000","last_unstale":"2024-04-05T02:50:32.931029+0000","last_undegraded":"2024-04-05T02:50:32.931029+0000","last_fullsized":"2024-04-05T02:50:32.931029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:49:35.432562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.422076+0000","last_change":"2024-04-05T02:49:47.208253+0000","last_active":"2024-04-05T02:49:50.422076+0000","last_peered":"2024-04-05T02:49:50.422076+0000","last_clean":"2024-04-05T02:49:50.422076+0000","last_became_active":"2024-04-05T02:49:47.207967+0000","last_became_peered":"2024-04-05T02:49:47.207967+0000","last_unstale":"2024-04-05T02:49:50.422076+0000","last_undegraded":"2024-04-05T02:49:50.422076+0000","last_fullsized":"2024-04-05T02:49:50.422076+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:48:22.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.913798+0000","last_change":"2024-04-05T02:49:47.188521+0000","last_active":"2024-04-05T02:49:58.913798+0000","last_peered":"2024-04-05T02:49:58.913798+0000","last_clean":"2024-04-05T02:49:58.913798+0000","last_became_active":"2024-04-05T02:49:47.188231+0000","last_became_peered":"2024-04-05T02:49:47.188231+0000","last_unstale":"2024-04-05T02:49:58.913798+0000","last_undegraded":"2024-04-05T02:49:58.913798+0000","last_fullsized":"2024-04-05T02:49:58.913798+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:49:13.887883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931533+0000","last_change":"2024-04-05T02:49:48.245097+0000","last_active":"2024-04-05T02:50:32.931533+0000","last_peered":"2024-04-05T02:50:32.931533+0000","last_clean":"2024-04-05T02:50:32.931533+0000","last_became_active":"2024-04-05T02:49:48.244750+0000","last_became_peered":"2024-04-05T02:49:48.244750+0000","last_unstale":"2024-04-05T02:50:32.931533+0000","last_undegraded":"2024-04-05T02:50:32.931533+0000","last_fullsized":"2024-04-05T02:50:32.931533+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:42:16.800839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931166+0000","last_change":"2024-04-05T02:49:47.196780+0000","last_active":"2024-04-05T02:50:32.931166+0000","last_peered":"2024-04-05T02:50:32.931166+0000","last_clean":"2024-04-05T02:50:32.931166+0000","last_became_active":"2024-04-05T02:49:47.196491+0000","last_became_peered":"2024-04-05T02:49:47.196491+0000","last_unstale":"2024-04-05T02:50:32.931166+0000","last_undegraded":"2024-04-05T02:50:32.931166+0000","last_fullsized":"2024-04-05T02:50:32.931166+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:30:05.059541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928592+0000","last_change":"2024-04-05T02:49:48.245084+0000","last_active":"2024-04-05T02:50:32.928592+0000","last_peered":"2024-04-05T02:50:32.928592+0000","last_clean":"2024-04-05T02:50:32.928592+0000","last_became_active":"2024-04-05T02:49:48.244766+0000","last_became_peered":"2024-04-05T02:49:48.244766+0000","last_unstale":"2024-04-05T02:50:32.928592+0000","last_undegraded":"2024-04-05T02:50:32.928592+0000","last_fullsized":"2024-04-05T02:50:32.928592+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:58:40.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930382+0000","last_change":"2024-04-05T02:49:47.189264+0000","last_active":"2024-04-05T02:50:32.930382+0000","last_peered":"2024-04-05T02:50:32.930382+0000","last_clean":"2024-04-05T02:50:32.930382+0000","last_became_active":"2024-04-05T02:49:47.188944+0000","last_became_peered":"2024-04-05T02:49:47.188944+0000","last_unstale":"2024-04-05T02:50:32.930382+0000","last_undegraded":"2024-04-05T02:50:32.930382+0000","last_fullsized":"2024-04-05T02:50:32.930382+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:29:15.760759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928527+0000","last_change":"2024-04-05T02:49:48.245246+0000","last_active":"2024-04-05T02:50:32.928527+0000","last_peered":"2024-04-05T02:50:32.928527+0000","last_clean":"2024-04-05T02:50:32.928527+0000","last_became_active":"2024-04-05T02:49:48.244713+0000","last_became_peered":"2024-04-05T02:49:48.244713+0000","last_unstale":"2024-04-05T02:50:32.928527+0000","last_undegraded":"2024-04-05T02:50:32.928527+0000","last_fullsized":"2024-04-05T02:50:32.928527+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:47:33.980664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203658+0000","last_change":"2024-04-05T02:49:47.198524+0000","last_active":"2024-04-05T02:49:49.203658+0000","last_peered":"2024-04-05T02:49:49.203658+0000","last_clean":"2024-04-05T02:49:49.203658+0000","last_became_active":"2024-04-05T02:49:47.198036+0000","last_became_peered":"2024-04-05T02:49:47.198036+0000","last_unstale":"2024-04-05T02:49:49.203658+0000","last_undegraded":"2024-04-05T02:49:49.203658+0000","last_fullsized":"2024-04-05T02:49:49.203658+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:13:44.697687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933204+0000","last_change":"2024-04-05T02:49:47.208079+0000","last_active":"2024-04-05T02:50:32.933204+0000","last_peered":"2024-04-05T02:50:32.933204+0000","last_clean":"2024-04-05T02:50:32.933204+0000","last_became_active":"2024-04-05T02:49:47.207741+0000","last_became_peered":"2024-04-05T02:49:47.207741+0000","last_unstale":"2024-04-05T02:50:32.933204+0000","last_undegraded":"2024-04-05T02:50:32.933204+0000","last_fullsized":"2024-04-05T02:50:32.933204+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:05:02.545819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928540+0000","last_change":"2024-04-05T02:49:47.198831+0000","last_active":"2024-04-05T02:50:32.928540+0000","last_peered":"2024-04-05T02:50:32.928540+0000","last_clean":"2024-04-05T02:50:32.928540+0000","last_became_active":"2024-04-05T02:49:47.198570+0000","last_became_peered":"2024-04-05T02:49:47.198570+0000","last_unstale":"2024-04-05T02:50:32.928540+0000","last_undegraded":"2024-04-05T02:50:32.928540+0000","last_fullsized":"2024-04-05T02:50:32.928540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:49:16.622423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203150+0000","last_change":"2024-04-05T02:49:48.221939+0000","last_active":"2024-04-05T02:49:49.203150+0000","last_peered":"2024-04-05T02:49:49.203150+0000","last_clean":"2024-04-05T02:49:49.203150+0000","last_became_active":"2024-04-05T02:49:48.221603+0000","last_became_peered":"2024-04-05T02:49:48.221603+0000","last_unstale":"2024-04-05T02:49:49.203150+0000","last_undegraded":"2024-04-05T02:49:49.203150+0000","last_fullsized":"2024-04-05T02:49:49.203150+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:50:24.489229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931420+0000","last_change":"2024-04-05T02:49:47.198624+0000","last_active":"2024-04-05T02:50:32.931420+0000","last_peered":"2024-04-05T02:50:32.931420+0000","last_clean":"2024-04-05T02:50:32.931420+0000","last_became_active":"2024-04-05T02:49:47.198354+0000","last_became_peered":"2024-04-05T02:49:47.198354+0000","last_unstale":"2024-04-05T02:50:32.931420+0000","last_undegraded":"2024-04-05T02:50:32.931420+0000","last_fullsized":"2024-04-05T02:50:32.931420+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:14:26.367910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928932+0000","last_change":"2024-04-05T02:49:48.245923+0000","last_active":"2024-04-05T02:50:32.928932+0000","last_peered":"2024-04-05T02:50:32.928932+0000","last_clean":"2024-04-05T02:50:32.928932+0000","last_became_active":"2024-04-05T02:49:48.245629+0000","last_became_peered":"2024-04-05T02:49:48.245629+0000","last_unstale":"2024-04-05T02:50:32.928932+0000","last_undegraded":"2024-04-05T02:50:32.928932+0000","last_fullsized":"2024-04-05T02:50:32.928932+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:39:09.813927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.908457+0000","last_change":"2024-04-05T02:49:47.194662+0000","last_active":"2024-04-05T02:49:58.908457+0000","last_peered":"2024-04-05T02:49:58.908457+0000","last_clean":"2024-04-05T02:49:58.908457+0000","last_became_active":"2024-04-05T02:49:47.194047+0000","last_became_peered":"2024-04-05T02:49:47.194047+0000","last_unstale":"2024-04-05T02:49:58.908457+0000","last_undegraded":"2024-04-05T02:49:58.908457+0000","last_fullsized":"2024-04-05T02:49:58.908457+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:00:36.223513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298676,"kb_used_data":1316,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446476,"statfs":{"total":95995035648,"available":95689191424,"internally_reserved":0,"allocated":1347584,"data_stored":908552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.53300000000000003,"15min":0.52800000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74099999999999999,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.45400000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.378,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68799999999999994,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.443},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.48699999999999999}]},{"osd":2,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73499999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.40899999999999997,"5min":0.371,"15min":0.371},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.40799999999999997}]},{"osd":3,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.42099999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72399999999999998,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.39600000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.47599999999999998}]},{"osd":4,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.5,"15min":0.5},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.83699999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.51500000000000001,"15min":0.51000000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.38700000000000001}]},{"osd":6,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.496}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298664,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446488,"statfs":{"total":95995035648,"available":95689203712,"internally_reserved":0,"allocated":1335296,"data_stored":908406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.502,"15min":0.49399999999999999},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.501,"15min":0.499},"min":{"1min":0.33500000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64200000000000002,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.61199999999999999}]},{"osd":1,"last update":"Fri Apr 5 02:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.503,"5min":0.499,"15min":0.498},"min":{"1min":0.34999999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.65800000000000003}]},{"osd":2,"last update":"Fri Apr 5 02:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.48699999999999999,"15min":0.47899999999999998},"min":{"1min":0.38,"5min":0.314,"15min":0.314},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.52700000000000002,"15min":0.52500000000000002},"min":{"1min":0.36899999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.67700000000000005}]},{"osd":3,"last update":"Fri Apr 5 02:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.39400000000000002,"5min":0.375,"15min":0.375},"max":{"1min":0.72799999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.50800000000000001,"15min":0.504},"min":{"1min":0.41399999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.57899999999999996}]},{"osd":4,"last update":"Fri Apr 5 02:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.38500000000000001,"15min":0.375},"min":{"1min":0.309,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.40600000000000003,"15min":0.39500000000000002},"min":{"1min":0.27500000000000002,"5min":0.252,"15min":0.252},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.70599999999999996}]},{"osd":5,"last update":"Fri Apr 5 02:49:39 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.42999999999999999,"15min":0.41799999999999998},"min":{"1min":0.36899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.64200000000000002,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.46700000000000003,"15min":0.45100000000000001},"min":{"1min":0.35799999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.78800000000000003}]},{"osd":7,"last update":"Fri Apr 5 02:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.48199999999999998,"15min":0.47999999999999998},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.624,"5min":0.751,"15min":0.751},"last":0.755},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.51700000000000002,"15min":0.51100000000000001},"min":{"1min":0.40400000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71099999999999997,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.72999999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":316151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.371,"15min":0.32800000000000001},"min":{"1min":0.26700000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.53800000000000003,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.443},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39600000000000002,"15min":0.34899999999999998},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.67800000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.30099999999999999}]},{"osd":2,"last update":"Fri Apr 5 02:50:04 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.376,"15min":0.373},"min":{"1min":0.29199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41299999999999998,"15min":0.39500000000000002},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.57899999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.40300000000000002}]},{"osd":3,"last update":"Fri Apr 5 02:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.46100000000000002,"15min":0.47899999999999998},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.5,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.42599999999999999,"15min":0.44600000000000001},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":0.58599999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.27900000000000003}]},{"osd":4,"last update":"Fri Apr 5 02:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.61499999999999999,"15min":0.629},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.88300000000000001,"5min":1.022,"15min":1.022},"last":0.374},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58299999999999996,"15min":0.59099999999999997},"min":{"1min":0.40000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.56699999999999995}]},{"osd":5,"last update":"Fri Apr 5 02:50:15 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.56699999999999995,"15min":0.57499999999999996},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60899999999999999,"15min":0.624},"min":{"1min":0.42999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.85899999999999999,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.58499999999999996}]},{"osd":6,"last update":"Fri Apr 5 02:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.55000000000000004},"min":{"1min":0.44,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.59099999999999997,"15min":0.58799999999999997},"min":{"1min":0.46999999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.89900000000000002,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.52100000000000002}]},{"osd":7,"last update":"Fri Apr 5 02:50:04 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.629,"5min":0.56799999999999995,"15min":0.55800000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.59299999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1327104,"data_stored":907376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 02:50:43 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.35199999999999998,"15min":0.32400000000000001},"min":{"1min":0.22,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.501,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.34899999999999998},"min":{"1min":0.26100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.52900000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.51300000000000001}]},{"osd":2,"last update":"Fri Apr 5 02:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.41599999999999998,"15min":0.438},"min":{"1min":0.27300000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.52400000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.37,"5min":0.39800000000000002,"15min":0.39900000000000002},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.48899999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.501}]},{"osd":3,"last update":"Fri Apr 5 02:50:31 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.434,"15min":0.45000000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.51500000000000001,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53600000000000003,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.53600000000000003}]},{"osd":4,"last update":"Fri Apr 5 02:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.40600000000000003,"5min":0.377,"15min":0.377},"max":{"1min":0.623,"5min":0.875,"15min":0.875},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54700000000000004,"15min":0.53700000000000003},"min":{"1min":0.42199999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.68000000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.67300000000000004}]},{"osd":5,"last update":"Fri Apr 5 02:50:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.41899999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79000000000000004,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.625},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.58599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79100000000000004,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.73999999999999999}]},{"osd":6,"last update":"Fri Apr 5 02:50:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60899999999999999,"15min":0.627},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76000000000000001,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61499999999999999,"15min":0.63300000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82199999999999995,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.65800000000000003}]},{"osd":7,"last update":"Fri Apr 5 02:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.45800000000000002,"5min":0.435,"15min":0.435},"max":{"1min":0.752,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.622,"15min":0.628},"min":{"1min":0.48599999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76800000000000002,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.70399999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.443},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.39300000000000002,"15min":0.39100000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.48799999999999999,"5min":0.62,"15min":0.62},"last":0.38600000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.28199999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.61899999999999999,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.44900000000000001,"15min":0.42899999999999999},"min":{"1min":0.318,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52900000000000003,"5min":0.751,"15min":0.751},"last":0.46899999999999997}]},{"osd":3,"last update":"Fri Apr 5 02:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48999999999999999,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42999999999999999,"15min":0.45800000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.327,"5min":1.327,"15min":1.327},"last":0.42299999999999999}]},{"osd":4,"last update":"Fri Apr 5 02:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.60199999999999998,"15min":0.60399999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.77100000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59299999999999997,"15min":0.61499999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77200000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.73799999999999999}]},{"osd":5,"last update":"Fri Apr 5 02:50:12 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57099999999999995,"15min":0.58299999999999996},"min":{"1min":0.372,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73399999999999999,"5min":0.745,"15min":0.745},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.47599999999999998,"5min":0.443,"15min":0.443},"max":{"1min":0.83199999999999996,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.72799999999999998}]},{"osd":6,"last update":"Fri Apr 5 02:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.63700000000000001,"15min":0.63600000000000001},"min":{"1min":0.47899999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.59499999999999997,"15min":0.59199999999999997},"min":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":0.68500000000000005}]},{"osd":7,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.76100000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.751},{"interface":"front","average":{"1min":0.629,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.47499999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.70299999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":315918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.40100000000000002,"15min":0.39400000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.497,"5min":1.252,"15min":1.252},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.40400000000000003,"15min":0.40999999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48999999999999999,"5min":0.63,"15min":0.63},"last":0.48099999999999998}]},{"osd":1,"last update":"Fri Apr 5 02:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.41199999999999998,"15min":0.42199999999999999},"min":{"1min":0.248,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46899999999999997,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.374,"5min":0.433,"15min":0.432},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.46700000000000003,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.60499999999999998}]},{"osd":2,"last update":"Fri Apr 5 02:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"min":{"1min":0.313,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.55100000000000005,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.437,"15min":0.45100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.44400000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.58899999999999997}]},{"osd":4,"last update":"Fri Apr 5 02:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58999999999999997,"15min":0.59299999999999997},"min":{"1min":0.41799999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.80100000000000005,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.749},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58999999999999997,"15min":0.58199999999999996},"min":{"1min":0.44400000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.98699999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.73599999999999999}]},{"osd":5,"last update":"Fri Apr 5 02:50:12 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.61199999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.80900000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.747,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.71499999999999997}]},{"osd":6,"last update":"Fri Apr 5 02:50:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69799999999999995,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64400000000000002,"15min":0.64000000000000001},"min":{"1min":0.45900000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83099999999999996,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.83099999999999996}]},{"osd":7,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.625,"15min":0.629},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.78400000000000003,"5min":0.88,"15min":0.88},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.65500000000000003,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.84799999999999998,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.88300000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298088,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447064,"statfs":{"total":95995035648,"available":95689793536,"internally_reserved":0,"allocated":749568,"data_stored":318114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.49299999999999999,"15min":0.5},"min":{"1min":0.25700000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.60799999999999998,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.5,"5min":0.505,"15min":0.495},"min":{"1min":0.35999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.66000000000000003,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.60699999999999998}]},{"osd":1,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.33400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.58999999999999997,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.59499999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.45700000000000002}]},{"osd":2,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.502,"15min":0.496},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.498,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70199999999999996,"5min":0.747,"15min":0.747},"last":0.51200000000000001}]},{"osd":3,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.53900000000000003},"min":{"1min":0.375,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69499999999999995,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.54800000000000004,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.37,"15min":0.37},"max":{"1min":0.66100000000000003,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.64300000000000002}]},{"osd":5,"last update":"Fri Apr 5 02:50:14 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.437,"15min":0.42299999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.68799999999999994,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.432,"15min":0.41199999999999998},"min":{"1min":0.32700000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.67500000000000004,"5min":0.754,"15min":0.754},"last":0.68500000000000005}]},{"osd":6,"last update":"Fri Apr 5 02:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.47999999999999998,"15min":0.46100000000000002},"min":{"1min":0.35499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.72499999999999998,"5min":0.755,"15min":0.755},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.495,"15min":0.47299999999999998},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.72199999999999998}]},{"osd":7,"last update":"Fri Apr 5 02:50:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.437,"5min":0.316,"15min":0.316},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.70599999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":724992,"data_stored":316468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.55800000000000005},"min":{"1min":0.41299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":0.66300000000000003,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.51900000000000002,"15min":0.53400000000000003},"min":{"1min":0.35699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.753,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.57899999999999996}]},{"osd":1,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.73099999999999998,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.38300000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.69499999999999995,"5min":0.745,"15min":0.745},"last":0.59999999999999998}]},{"osd":2,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.52300000000000002,"15min":0.53100000000000003},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.67400000000000004,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.63},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55900000000000005,"15min":0.57399999999999995},"min":{"1min":0.41599999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.71199999999999997,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56399999999999995}]},{"osd":3,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70899999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72999999999999998,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.66900000000000004}]},{"osd":4,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.44700000000000001,"15min":0.434},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.751,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.44800000000000001,"15min":0.433},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66800000000000004,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.42799999999999999}]},{"osd":6,"last update":"Fri Apr 5 02:50:42 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.501,"15min":0.48899999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.501,"15min":0.49399999999999999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69899999999999995,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Fri Apr 5 02:50:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.374,"5min":0.373,"15min":0.373},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.502,"5min":0.54500000000000004,"15min":0.55200000000000005},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.69199999999999995,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.505}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T02:50:46.950 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T02:50:47.207 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:47.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:46 smithi150 ceph-mon[29558]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:50:47.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:46 smithi150 ceph-mon[41818]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:50:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:46 smithi196 ceph-mon[33169]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:50:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:47 smithi150 ceph-mon[29558]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:47 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:48.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:47 smithi150 ceph-mon[41818]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:48.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:47 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:47 smithi196 ceph-mon[33169]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:47 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:50:49.108 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:50:49.109 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-04-05T02:50:49.939 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":82,"stamp":"2024-04-05T02:50:48.853050+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386352,"kb_used_data":7480,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574864,"statfs":{"total":767960285184,"available":765516660736,"internally_reserved":0,"allocated":7659520,"data_stored":4308005,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002664"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.420910+0000","last_change":"2024-04-05T02:49:47.206413+0000","last_active":"2024-04-05T02:49:50.420910+0000","last_peered":"2024-04-05T02:49:50.420910+0000","last_clean":"2024-04-05T02:49:50.420910+0000","last_became_active":"2024-04-05T02:49:47.206099+0000","last_became_peered":"2024-04-05T02:49:47.206099+0000","last_unstale":"2024-04-05T02:49:50.420910+0000","last_undegraded":"2024-04-05T02:49:50.420910+0000","last_fullsized":"2024-04-05T02:49:50.420910+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:36:28.371339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":17,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.912568+0000","last_change":"2024-04-05T02:49:47.198309+0000","last_active":"2024-04-05T02:49:58.912568+0000","last_peered":"2024-04-05T02:49:58.912568+0000","last_clean":"2024-04-05T02:49:58.912568+0000","last_became_active":"2024-04-05T02:49:47.197378+0000","last_became_peered":"2024-04-05T02:49:47.197378+0000","last_unstale":"2024-04-05T02:49:58.912568+0000","last_undegraded":"2024-04-05T02:49:58.912568+0000","last_fullsized":"2024-04-05T02:49:58.912568+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:44:03.975757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928554+0000","last_change":"2024-04-05T02:49:48.245928+0000","last_active":"2024-04-05T02:50:32.928554+0000","last_peered":"2024-04-05T02:50:32.928554+0000","last_clean":"2024-04-05T02:50:32.928554+0000","last_became_active":"2024-04-05T02:49:48.245582+0000","last_became_peered":"2024-04-05T02:49:48.245582+0000","last_unstale":"2024-04-05T02:50:32.928554+0000","last_undegraded":"2024-04-05T02:50:32.928554+0000","last_fullsized":"2024-04-05T02:50:32.928554+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:19:54.560500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929051+0000","last_change":"2024-04-05T02:49:47.194809+0000","last_active":"2024-04-05T02:50:32.929051+0000","last_peered":"2024-04-05T02:50:32.929051+0000","last_clean":"2024-04-05T02:50:32.929051+0000","last_became_active":"2024-04-05T02:49:47.194535+0000","last_became_peered":"2024-04-05T02:49:47.194535+0000","last_unstale":"2024-04-05T02:50:32.929051+0000","last_undegraded":"2024-04-05T02:50:32.929051+0000","last_fullsized":"2024-04-05T02:50:32.929051+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:26:50.573010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421291+0000","last_change":"2024-04-05T02:49:47.247249+0000","last_active":"2024-04-05T02:49:50.421291+0000","last_peered":"2024-04-05T02:49:50.421291+0000","last_clean":"2024-04-05T02:49:50.421291+0000","last_became_active":"2024-04-05T02:49:47.246981+0000","last_became_peered":"2024-04-05T02:49:47.246981+0000","last_unstale":"2024-04-05T02:49:50.421291+0000","last_undegraded":"2024-04-05T02:49:50.421291+0000","last_fullsized":"2024-04-05T02:49:50.421291+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:57:52.416336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930443+0000","last_change":"2024-04-05T02:49:47.188962+0000","last_active":"2024-04-05T02:50:32.930443+0000","last_peered":"2024-04-05T02:50:32.930443+0000","last_clean":"2024-04-05T02:50:32.930443+0000","last_became_active":"2024-04-05T02:49:47.188681+0000","last_became_peered":"2024-04-05T02:49:47.188681+0000","last_unstale":"2024-04-05T02:50:32.930443+0000","last_undegraded":"2024-04-05T02:50:32.930443+0000","last_fullsized":"2024-04-05T02:50:32.930443+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:59:25.089268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421985+0000","last_change":"2024-04-05T02:49:47.248544+0000","last_active":"2024-04-05T02:49:50.421985+0000","last_peered":"2024-04-05T02:49:50.421985+0000","last_clean":"2024-04-05T02:49:50.421985+0000","last_became_active":"2024-04-05T02:49:47.248214+0000","last_became_peered":"2024-04-05T02:49:47.248214+0000","last_unstale":"2024-04-05T02:49:50.421985+0000","last_undegraded":"2024-04-05T02:49:50.421985+0000","last_fullsized":"2024-04-05T02:49:50.421985+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:16:55.500018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930286+0000","last_change":"2024-04-05T02:49:47.191401+0000","last_active":"2024-04-05T02:50:32.930286+0000","last_peered":"2024-04-05T02:50:32.930286+0000","last_clean":"2024-04-05T02:50:32.930286+0000","last_became_active":"2024-04-05T02:49:47.190758+0000","last_became_peered":"2024-04-05T02:49:47.190758+0000","last_unstale":"2024-04-05T02:50:32.930286+0000","last_undegraded":"2024-04-05T02:50:32.930286+0000","last_fullsized":"2024-04-05T02:50:32.930286+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:24.309425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-05T02:50:33.949153+0000","last_change":"2024-04-05T02:50:33.949153+0000","last_active":"2024-04-05T02:50:33.949153+0000","last_peered":"2024-04-05T02:50:33.949153+0000","last_clean":"2024-04-05T02:50:33.949153+0000","last_became_active":"2024-04-05T02:50:33.948818+0000","last_became_peered":"2024-04-05T02:50:33.948818+0000","last_unstale":"2024-04-05T02:50:33.949153+0000","last_undegraded":"2024-04-05T02:50:33.949153+0000","last_fullsized":"2024-04-05T02:50:33.949153+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:53.952889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.907919+0000","last_change":"2024-04-05T02:49:47.195429+0000","last_active":"2024-04-05T02:49:58.907919+0000","last_peered":"2024-04-05T02:49:58.907919+0000","last_clean":"2024-04-05T02:49:58.907919+0000","last_became_active":"2024-04-05T02:49:47.195063+0000","last_became_peered":"2024-04-05T02:49:47.195063+0000","last_unstale":"2024-04-05T02:49:58.907919+0000","last_undegraded":"2024-04-05T02:49:58.907919+0000","last_fullsized":"2024-04-05T02:49:58.907919+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:23:16.328321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421875+0000","last_change":"2024-04-05T02:49:47.206433+0000","last_active":"2024-04-05T02:49:50.421875+0000","last_peered":"2024-04-05T02:49:50.421875+0000","last_clean":"2024-04-05T02:49:50.421875+0000","last_became_active":"2024-04-05T02:49:47.206035+0000","last_became_peered":"2024-04-05T02:49:47.206035+0000","last_unstale":"2024-04-05T02:49:50.421875+0000","last_undegraded":"2024-04-05T02:49:50.421875+0000","last_fullsized":"2024-04-05T02:49:50.421875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:08:55.149404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:50:03.303344+0000","last_change":"2024-04-05T02:49:47.200921+0000","last_active":"2024-04-05T02:50:03.303344+0000","last_peered":"2024-04-05T02:50:03.303344+0000","last_clean":"2024-04-05T02:50:03.303344+0000","last_became_active":"2024-04-05T02:49:47.200539+0000","last_became_peered":"2024-04-05T02:49:47.200539+0000","last_unstale":"2024-04-05T02:50:03.303344+0000","last_undegraded":"2024-04-05T02:50:03.303344+0000","last_fullsized":"2024-04-05T02:50:03.303344+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:43:09.109344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933266+0000","last_change":"2024-04-05T02:48:35.842380+0000","last_active":"2024-04-05T02:50:32.933266+0000","last_peered":"2024-04-05T02:50:32.933266+0000","last_clean":"2024-04-05T02:50:32.933266+0000","last_became_active":"2024-04-05T02:48:35.842036+0000","last_became_peered":"2024-04-05T02:48:35.842036+0000","last_unstale":"2024-04-05T02:50:32.933266+0000","last_undegraded":"2024-04-05T02:50:32.933266+0000","last_fullsized":"2024-04-05T02:50:32.933266+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:53:50.319270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928433+0000","last_change":"2024-04-05T02:49:48.222222+0000","last_active":"2024-04-05T02:50:32.928433+0000","last_peered":"2024-04-05T02:50:32.928433+0000","last_clean":"2024-04-05T02:50:32.928433+0000","last_became_active":"2024-04-05T02:49:48.221172+0000","last_became_peered":"2024-04-05T02:49:48.221172+0000","last_unstale":"2024-04-05T02:50:32.928433+0000","last_undegraded":"2024-04-05T02:50:32.928433+0000","last_fullsized":"2024-04-05T02:50:32.928433+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:17:29.162596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.932073+0000","last_change":"2024-04-05T02:48:36.012430+0000","last_active":"2024-04-05T02:50:32.932073+0000","last_peered":"2024-04-05T02:50:32.932073+0000","last_clean":"2024-04-05T02:50:32.932073+0000","last_became_active":"2024-04-05T02:48:36.012069+0000","last_became_peered":"2024-04-05T02:48:36.012069+0000","last_unstale":"2024-04-05T02:50:32.932073+0000","last_undegraded":"2024-04-05T02:50:32.932073+0000","last_fullsized":"2024-04-05T02:50:32.932073+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T02:59:27.856396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929113+0000","last_change":"2024-04-05T02:49:47.199782+0000","last_active":"2024-04-05T02:50:32.929113+0000","last_peered":"2024-04-05T02:50:32.929113+0000","last_clean":"2024-04-05T02:50:32.929113+0000","last_became_active":"2024-04-05T02:49:47.198795+0000","last_became_peered":"2024-04-05T02:49:47.198795+0000","last_unstale":"2024-04-05T02:50:32.929113+0000","last_undegraded":"2024-04-05T02:50:32.929113+0000","last_fullsized":"2024-04-05T02:50:32.929113+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:41:38.102537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203302+0000","last_change":"2024-04-05T02:49:48.222221+0000","last_active":"2024-04-05T02:49:49.203302+0000","last_peered":"2024-04-05T02:49:49.203302+0000","last_clean":"2024-04-05T02:49:49.203302+0000","last_became_active":"2024-04-05T02:49:48.221228+0000","last_became_peered":"2024-04-05T02:49:48.221228+0000","last_unstale":"2024-04-05T02:49:49.203302+0000","last_undegraded":"2024-04-05T02:49:49.203302+0000","last_fullsized":"2024-04-05T02:49:49.203302+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:30:17.668761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931785+0000","last_change":"2024-04-05T02:49:47.197146+0000","last_active":"2024-04-05T02:50:32.931785+0000","last_peered":"2024-04-05T02:50:32.931785+0000","last_clean":"2024-04-05T02:50:32.931785+0000","last_became_active":"2024-04-05T02:49:47.194951+0000","last_became_peered":"2024-04-05T02:49:47.194951+0000","last_unstale":"2024-04-05T02:50:32.931785+0000","last_undegraded":"2024-04-05T02:50:32.931785+0000","last_fullsized":"2024-04-05T02:50:32.931785+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:19:07.376060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931221+0000","last_change":"2024-04-05T02:48:36.033701+0000","last_active":"2024-04-05T02:50:32.931221+0000","last_peered":"2024-04-05T02:50:32.931221+0000","last_clean":"2024-04-05T02:50:32.931221+0000","last_became_active":"2024-04-05T02:48:36.033383+0000","last_became_peered":"2024-04-05T02:48:36.033383+0000","last_unstale":"2024-04-05T02:50:32.931221+0000","last_undegraded":"2024-04-05T02:50:32.931221+0000","last_fullsized":"2024-04-05T02:50:32.931221+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:57:58.141349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203246+0000","last_change":"2024-04-05T02:49:48.246020+0000","last_active":"2024-04-05T02:49:49.203246+0000","last_peered":"2024-04-05T02:49:49.203246+0000","last_clean":"2024-04-05T02:49:49.203246+0000","last_became_active":"2024-04-05T02:49:48.245722+0000","last_became_peered":"2024-04-05T02:49:48.245722+0000","last_unstale":"2024-04-05T02:49:49.203246+0000","last_undegraded":"2024-04-05T02:49:49.203246+0000","last_fullsized":"2024-04-05T02:49:49.203246+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:20:20.342453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.202995+0000","last_change":"2024-04-05T02:49:47.192884+0000","last_active":"2024-04-05T02:49:49.202995+0000","last_peered":"2024-04-05T02:49:49.202995+0000","last_clean":"2024-04-05T02:49:49.202995+0000","last_became_active":"2024-04-05T02:49:47.192527+0000","last_became_peered":"2024-04-05T02:49:47.192527+0000","last_unstale":"2024-04-05T02:49:49.202995+0000","last_undegraded":"2024-04-05T02:49:49.202995+0000","last_fullsized":"2024-04-05T02:49:49.202995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:58:54.090853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931093+0000","last_change":"2024-04-05T02:48:36.033673+0000","last_active":"2024-04-05T02:50:32.931093+0000","last_peered":"2024-04-05T02:50:32.931093+0000","last_clean":"2024-04-05T02:50:32.931093+0000","last_became_active":"2024-04-05T02:48:36.033304+0000","last_became_peered":"2024-04-05T02:48:36.033304+0000","last_unstale":"2024-04-05T02:50:32.931093+0000","last_undegraded":"2024-04-05T02:50:32.931093+0000","last_fullsized":"2024-04-05T02:50:32.931093+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:28:26.516492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933230+0000","last_change":"2024-04-05T02:49:47.246904+0000","last_active":"2024-04-05T02:50:32.933230+0000","last_peered":"2024-04-05T02:50:32.933230+0000","last_clean":"2024-04-05T02:50:32.933230+0000","last_became_active":"2024-04-05T02:49:47.246606+0000","last_became_peered":"2024-04-05T02:49:47.246606+0000","last_unstale":"2024-04-05T02:50:32.933230+0000","last_undegraded":"2024-04-05T02:50:32.933230+0000","last_fullsized":"2024-04-05T02:50:32.933230+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:18:33.167051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928805+0000","last_change":"2024-04-05T02:49:47.196017+0000","last_active":"2024-04-05T02:50:32.928805+0000","last_peered":"2024-04-05T02:50:32.928805+0000","last_clean":"2024-04-05T02:50:32.928805+0000","last_became_active":"2024-04-05T02:49:47.195723+0000","last_became_peered":"2024-04-05T02:49:47.195723+0000","last_unstale":"2024-04-05T02:50:32.928805+0000","last_undegraded":"2024-04-05T02:50:32.928805+0000","last_fullsized":"2024-04-05T02:50:32.928805+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:35:20.900943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":273,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931029+0000","last_change":"2024-04-05T02:48:04.212692+0000","last_active":"2024-04-05T02:50:32.931029+0000","last_peered":"2024-04-05T02:50:32.931029+0000","last_clean":"2024-04-05T02:50:32.931029+0000","last_became_active":"2024-04-05T02:48:04.212212+0000","last_became_peered":"2024-04-05T02:48:04.212212+0000","last_unstale":"2024-04-05T02:50:32.931029+0000","last_undegraded":"2024-04-05T02:50:32.931029+0000","last_fullsized":"2024-04-05T02:50:32.931029+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_clean_scrub_stamp":"2024-04-05T02:45:59.919732+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:33:13.395158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930145+0000","last_change":"2024-04-05T02:48:35.842260+0000","last_active":"2024-04-05T02:50:32.930145+0000","last_peered":"2024-04-05T02:50:32.930145+0000","last_clean":"2024-04-05T02:50:32.930145+0000","last_became_active":"2024-04-05T02:48:35.841968+0000","last_became_peered":"2024-04-05T02:48:35.841968+0000","last_unstale":"2024-04-05T02:50:32.930145+0000","last_undegraded":"2024-04-05T02:50:32.930145+0000","last_fullsized":"2024-04-05T02:50:32.930145+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:37:28.928023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931588+0000","last_change":"2024-04-05T02:49:48.209094+0000","last_active":"2024-04-05T02:50:32.931588+0000","last_peered":"2024-04-05T02:50:32.931588+0000","last_clean":"2024-04-05T02:50:32.931588+0000","last_became_active":"2024-04-05T02:49:48.208816+0000","last_became_peered":"2024-04-05T02:49:48.208816+0000","last_unstale":"2024-04-05T02:50:32.931588+0000","last_undegraded":"2024-04-05T02:50:32.931588+0000","last_fullsized":"2024-04-05T02:50:32.931588+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:49:54.730675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931401+0000","last_change":"2024-04-05T02:49:47.196981+0000","last_active":"2024-04-05T02:50:32.931401+0000","last_peered":"2024-04-05T02:50:32.931401+0000","last_clean":"2024-04-05T02:50:32.931401+0000","last_became_active":"2024-04-05T02:49:47.196253+0000","last_became_peered":"2024-04-05T02:49:47.196253+0000","last_unstale":"2024-04-05T02:50:32.931401+0000","last_undegraded":"2024-04-05T02:50:32.931401+0000","last_fullsized":"2024-04-05T02:50:32.931401+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:46:10.152538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930917+0000","last_change":"2024-04-05T02:48:36.033794+0000","last_active":"2024-04-05T02:50:32.930917+0000","last_peered":"2024-04-05T02:50:32.930917+0000","last_clean":"2024-04-05T02:50:32.930917+0000","last_became_active":"2024-04-05T02:48:36.033472+0000","last_became_peered":"2024-04-05T02:48:36.033472+0000","last_unstale":"2024-04-05T02:50:32.930917+0000","last_undegraded":"2024-04-05T02:50:32.930917+0000","last_fullsized":"2024-04-05T02:50:32.930917+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:39:42.977557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931715+0000","last_change":"2024-04-05T02:49:48.246485+0000","last_active":"2024-04-05T02:50:32.931715+0000","last_peered":"2024-04-05T02:50:32.931715+0000","last_clean":"2024-04-05T02:50:32.931715+0000","last_became_active":"2024-04-05T02:49:48.244873+0000","last_became_peered":"2024-04-05T02:49:48.244873+0000","last_unstale":"2024-04-05T02:50:32.931715+0000","last_undegraded":"2024-04-05T02:50:32.931715+0000","last_fullsized":"2024-04-05T02:50:32.931715+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:56:22.827371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.422071+0000","last_change":"2024-04-05T02:49:47.192710+0000","last_active":"2024-04-05T02:49:50.422071+0000","last_peered":"2024-04-05T02:49:50.422071+0000","last_clean":"2024-04-05T02:49:50.422071+0000","last_became_active":"2024-04-05T02:49:47.192355+0000","last_became_peered":"2024-04-05T02:49:47.192355+0000","last_unstale":"2024-04-05T02:49:50.422071+0000","last_undegraded":"2024-04-05T02:49:50.422071+0000","last_fullsized":"2024-04-05T02:49:50.422071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:39:34.153715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929998+0000","last_change":"2024-04-05T02:48:35.840743+0000","last_active":"2024-04-05T02:50:32.929998+0000","last_peered":"2024-04-05T02:50:32.929998+0000","last_clean":"2024-04-05T02:50:32.929998+0000","last_became_active":"2024-04-05T02:48:35.840424+0000","last_became_peered":"2024-04-05T02:48:35.840424+0000","last_unstale":"2024-04-05T02:50:32.929998+0000","last_undegraded":"2024-04-05T02:50:32.929998+0000","last_fullsized":"2024-04-05T02:50:32.929998+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:04:16.216370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931634+0000","last_change":"2024-04-05T02:49:48.246029+0000","last_active":"2024-04-05T02:50:32.931634+0000","last_peered":"2024-04-05T02:50:32.931634+0000","last_clean":"2024-04-05T02:50:32.931634+0000","last_became_active":"2024-04-05T02:49:48.245716+0000","last_became_peered":"2024-04-05T02:49:48.245716+0000","last_unstale":"2024-04-05T02:50:32.931634+0000","last_undegraded":"2024-04-05T02:50:32.931634+0000","last_fullsized":"2024-04-05T02:50:32.931634+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:07:34.775055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930416+0000","last_change":"2024-04-05T02:49:47.190160+0000","last_active":"2024-04-05T02:50:32.930416+0000","last_peered":"2024-04-05T02:50:32.930416+0000","last_clean":"2024-04-05T02:50:32.930416+0000","last_became_active":"2024-04-05T02:49:47.189855+0000","last_became_peered":"2024-04-05T02:49:47.189855+0000","last_unstale":"2024-04-05T02:50:32.930416+0000","last_undegraded":"2024-04-05T02:50:32.930416+0000","last_fullsized":"2024-04-05T02:50:32.930416+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:48:04.061243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931642+0000","last_change":"2024-04-05T02:48:36.034060+0000","last_active":"2024-04-05T02:50:32.931642+0000","last_peered":"2024-04-05T02:50:32.931642+0000","last_clean":"2024-04-05T02:50:32.931642+0000","last_became_active":"2024-04-05T02:48:36.033747+0000","last_became_peered":"2024-04-05T02:48:36.033747+0000","last_unstale":"2024-04-05T02:50:32.931642+0000","last_undegraded":"2024-04-05T02:50:32.931642+0000","last_fullsized":"2024-04-05T02:50:32.931642+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:16:53.864447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931671+0000","last_change":"2024-04-05T02:49:48.245229+0000","last_active":"2024-04-05T02:50:32.931671+0000","last_peered":"2024-04-05T02:50:32.931671+0000","last_clean":"2024-04-05T02:50:32.931671+0000","last_became_active":"2024-04-05T02:49:48.244920+0000","last_became_peered":"2024-04-05T02:49:48.244920+0000","last_unstale":"2024-04-05T02:50:32.931671+0000","last_undegraded":"2024-04-05T02:50:32.931671+0000","last_fullsized":"2024-04-05T02:50:32.931671+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:22:28.965318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928457+0000","last_change":"2024-04-05T02:49:47.196272+0000","last_active":"2024-04-05T02:50:32.928457+0000","last_peered":"2024-04-05T02:50:32.928457+0000","last_clean":"2024-04-05T02:50:32.928457+0000","last_became_active":"2024-04-05T02:49:47.195949+0000","last_became_peered":"2024-04-05T02:49:47.195949+0000","last_unstale":"2024-04-05T02:50:32.928457+0000","last_undegraded":"2024-04-05T02:50:32.928457+0000","last_fullsized":"2024-04-05T02:50:32.928457+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:14:30.814375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421240+0000","last_change":"2024-04-05T02:49:47.206746+0000","last_active":"2024-04-05T02:49:50.421240+0000","last_peered":"2024-04-05T02:49:50.421240+0000","last_clean":"2024-04-05T02:49:50.421240+0000","last_became_active":"2024-04-05T02:49:47.206386+0000","last_became_peered":"2024-04-05T02:49:47.206386+0000","last_unstale":"2024-04-05T02:49:50.421240+0000","last_undegraded":"2024-04-05T02:49:50.421240+0000","last_fullsized":"2024-04-05T02:49:50.421240+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T02:50:27.026708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421899+0000","last_change":"2024-04-05T02:49:47.187686+0000","last_active":"2024-04-05T02:49:50.421899+0000","last_peered":"2024-04-05T02:49:50.421899+0000","last_clean":"2024-04-05T02:49:50.421899+0000","last_became_active":"2024-04-05T02:49:47.187375+0000","last_became_peered":"2024-04-05T02:49:47.187375+0000","last_unstale":"2024-04-05T02:49:50.421899+0000","last_undegraded":"2024-04-05T02:49:50.421899+0000","last_fullsized":"2024-04-05T02:49:50.421899+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:21:13.280128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421765+0000","last_change":"2024-04-05T02:49:47.210374+0000","last_active":"2024-04-05T02:49:50.421765+0000","last_peered":"2024-04-05T02:49:50.421765+0000","last_clean":"2024-04-05T02:49:50.421765+0000","last_became_active":"2024-04-05T02:49:47.210128+0000","last_became_peered":"2024-04-05T02:49:47.210128+0000","last_unstale":"2024-04-05T02:49:50.421765+0000","last_undegraded":"2024-04-05T02:49:50.421765+0000","last_fullsized":"2024-04-05T02:49:50.421765+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:02:46.410428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933879+0000","last_change":"2024-04-05T02:49:47.189496+0000","last_active":"2024-04-05T02:50:32.933879+0000","last_peered":"2024-04-05T02:50:32.933879+0000","last_clean":"2024-04-05T02:50:32.933879+0000","last_became_active":"2024-04-05T02:49:47.189216+0000","last_became_peered":"2024-04-05T02:49:47.189216+0000","last_unstale":"2024-04-05T02:50:32.933879+0000","last_undegraded":"2024-04-05T02:50:32.933879+0000","last_fullsized":"2024-04-05T02:50:32.933879+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:28:03.749329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421077+0000","last_change":"2024-04-05T02:49:47.248235+0000","last_active":"2024-04-05T02:49:50.421077+0000","last_peered":"2024-04-05T02:49:50.421077+0000","last_clean":"2024-04-05T02:49:50.421077+0000","last_became_active":"2024-04-05T02:49:47.247680+0000","last_became_peered":"2024-04-05T02:49:47.247680+0000","last_unstale":"2024-04-05T02:49:50.421077+0000","last_undegraded":"2024-04-05T02:49:50.421077+0000","last_fullsized":"2024-04-05T02:49:50.421077+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:01:36.464869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928812+0000","last_change":"2024-04-05T02:49:47.197456+0000","last_active":"2024-04-05T02:50:32.928812+0000","last_peered":"2024-04-05T02:50:32.928812+0000","last_clean":"2024-04-05T02:50:32.928812+0000","last_became_active":"2024-04-05T02:49:47.197145+0000","last_became_peered":"2024-04-05T02:49:47.197145+0000","last_unstale":"2024-04-05T02:50:32.928812+0000","last_undegraded":"2024-04-05T02:50:32.928812+0000","last_fullsized":"2024-04-05T02:50:32.928812+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:06:00.892577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.202988+0000","last_change":"2024-04-05T02:49:48.247884+0000","last_active":"2024-04-05T02:49:49.202988+0000","last_peered":"2024-04-05T02:49:49.202988+0000","last_clean":"2024-04-05T02:49:49.202988+0000","last_became_active":"2024-04-05T02:49:48.247548+0000","last_became_peered":"2024-04-05T02:49:48.247548+0000","last_unstale":"2024-04-05T02:49:49.202988+0000","last_undegraded":"2024-04-05T02:49:49.202988+0000","last_fullsized":"2024-04-05T02:49:49.202988+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:49:21.723925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930273+0000","last_change":"2024-04-05T02:49:47.190298+0000","last_active":"2024-04-05T02:50:32.930273+0000","last_peered":"2024-04-05T02:50:32.930273+0000","last_clean":"2024-04-05T02:50:32.930273+0000","last_became_active":"2024-04-05T02:49:47.189872+0000","last_became_peered":"2024-04-05T02:49:47.189872+0000","last_unstale":"2024-04-05T02:50:32.930273+0000","last_undegraded":"2024-04-05T02:50:32.930273+0000","last_fullsized":"2024-04-05T02:50:32.930273+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:47:45.371257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-05T02:50:33.934126+0000","last_change":"2024-04-05T02:50:33.934126+0000","last_active":"2024-04-05T02:50:33.934126+0000","last_peered":"2024-04-05T02:50:33.934126+0000","last_clean":"2024-04-05T02:50:33.934126+0000","last_became_active":"2024-04-05T02:50:33.933725+0000","last_became_peered":"2024-04-05T02:50:33.933725+0000","last_unstale":"2024-04-05T02:50:33.934126+0000","last_undegraded":"2024-04-05T02:50:33.934126+0000","last_fullsized":"2024-04-05T02:50:33.934126+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:47:08.848659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931292+0000","last_change":"2024-04-05T02:49:47.188385+0000","last_active":"2024-04-05T02:50:32.931292+0000","last_peered":"2024-04-05T02:50:32.931292+0000","last_clean":"2024-04-05T02:50:32.931292+0000","last_became_active":"2024-04-05T02:49:47.188108+0000","last_became_peered":"2024-04-05T02:49:47.188108+0000","last_unstale":"2024-04-05T02:50:32.931292+0000","last_undegraded":"2024-04-05T02:50:32.931292+0000","last_fullsized":"2024-04-05T02:50:32.931292+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:01.691480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.421736+0000","last_change":"2024-04-05T02:49:47.246658+0000","last_active":"2024-04-05T02:49:50.421736+0000","last_peered":"2024-04-05T02:49:50.421736+0000","last_clean":"2024-04-05T02:49:50.421736+0000","last_became_active":"2024-04-05T02:49:47.246374+0000","last_became_peered":"2024-04-05T02:49:47.246374+0000","last_unstale":"2024-04-05T02:49:50.421736+0000","last_undegraded":"2024-04-05T02:49:50.421736+0000","last_fullsized":"2024-04-05T02:49:50.421736+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:19:09.248707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931368+0000","last_change":"2024-04-05T02:49:47.192997+0000","last_active":"2024-04-05T02:50:32.931368+0000","last_peered":"2024-04-05T02:50:32.931368+0000","last_clean":"2024-04-05T02:50:32.931368+0000","last_became_active":"2024-04-05T02:49:47.192662+0000","last_became_peered":"2024-04-05T02:49:47.192662+0000","last_unstale":"2024-04-05T02:50:32.931368+0000","last_undegraded":"2024-04-05T02:50:32.931368+0000","last_fullsized":"2024-04-05T02:50:32.931368+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:42:21.816507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929953+0000","last_change":"2024-04-05T02:49:47.246329+0000","last_active":"2024-04-05T02:50:32.929953+0000","last_peered":"2024-04-05T02:50:32.929953+0000","last_clean":"2024-04-05T02:50:32.929953+0000","last_became_active":"2024-04-05T02:49:47.245961+0000","last_became_peered":"2024-04-05T02:49:47.245961+0000","last_unstale":"2024-04-05T02:50:32.929953+0000","last_undegraded":"2024-04-05T02:50:32.929953+0000","last_fullsized":"2024-04-05T02:50:32.929953+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:45:18.083543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931468+0000","last_change":"2024-04-05T02:49:47.192903+0000","last_active":"2024-04-05T02:50:32.931468+0000","last_peered":"2024-04-05T02:50:32.931468+0000","last_clean":"2024-04-05T02:50:32.931468+0000","last_became_active":"2024-04-05T02:49:47.192626+0000","last_became_peered":"2024-04-05T02:49:47.192626+0000","last_unstale":"2024-04-05T02:50:32.931468+0000","last_undegraded":"2024-04-05T02:50:32.931468+0000","last_fullsized":"2024-04-05T02:50:32.931468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:48:07.433594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.929024+0000","last_change":"2024-04-05T02:49:48.222667+0000","last_active":"2024-04-05T02:50:32.929024+0000","last_peered":"2024-04-05T02:50:32.929024+0000","last_clean":"2024-04-05T02:50:32.929024+0000","last_became_active":"2024-04-05T02:49:48.221235+0000","last_became_peered":"2024-04-05T02:49:48.221235+0000","last_unstale":"2024-04-05T02:50:32.929024+0000","last_undegraded":"2024-04-05T02:50:32.929024+0000","last_fullsized":"2024-04-05T02:50:32.929024+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:11:26.797834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.932959+0000","last_change":"2024-04-05T02:49:47.188396+0000","last_active":"2024-04-05T02:50:32.932959+0000","last_peered":"2024-04-05T02:50:32.932959+0000","last_clean":"2024-04-05T02:50:32.932959+0000","last_became_active":"2024-04-05T02:49:47.188129+0000","last_became_peered":"2024-04-05T02:49:47.188129+0000","last_unstale":"2024-04-05T02:50:32.932959+0000","last_undegraded":"2024-04-05T02:50:32.932959+0000","last_fullsized":"2024-04-05T02:50:32.932959+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:07:47.941423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203056+0000","last_change":"2024-04-05T02:49:48.221773+0000","last_active":"2024-04-05T02:49:49.203056+0000","last_peered":"2024-04-05T02:49:49.203056+0000","last_clean":"2024-04-05T02:49:49.203056+0000","last_became_active":"2024-04-05T02:49:48.221402+0000","last_became_peered":"2024-04-05T02:49:48.221402+0000","last_unstale":"2024-04-05T02:49:49.203056+0000","last_undegraded":"2024-04-05T02:49:49.203056+0000","last_fullsized":"2024-04-05T02:49:49.203056+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:52:47.436908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930818+0000","last_change":"2024-04-05T02:49:47.190900+0000","last_active":"2024-04-05T02:50:32.930818+0000","last_peered":"2024-04-05T02:50:32.930818+0000","last_clean":"2024-04-05T02:50:32.930818+0000","last_became_active":"2024-04-05T02:49:47.190633+0000","last_became_peered":"2024-04-05T02:49:47.190633+0000","last_unstale":"2024-04-05T02:50:32.930818+0000","last_undegraded":"2024-04-05T02:50:32.930818+0000","last_fullsized":"2024-04-05T02:50:32.930818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:59:46.877238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930335+0000","last_change":"2024-04-05T02:49:47.248198+0000","last_active":"2024-04-05T02:50:32.930335+0000","last_peered":"2024-04-05T02:50:32.930335+0000","last_clean":"2024-04-05T02:50:32.930335+0000","last_became_active":"2024-04-05T02:49:47.247847+0000","last_became_peered":"2024-04-05T02:49:47.247847+0000","last_unstale":"2024-04-05T02:50:32.930335+0000","last_undegraded":"2024-04-05T02:50:32.930335+0000","last_fullsized":"2024-04-05T02:50:32.930335+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:39:30.162920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.202914+0000","last_change":"2024-04-05T02:49:47.191362+0000","last_active":"2024-04-05T02:49:49.202914+0000","last_peered":"2024-04-05T02:49:49.202914+0000","last_clean":"2024-04-05T02:49:49.202914+0000","last_became_active":"2024-04-05T02:49:47.191040+0000","last_became_peered":"2024-04-05T02:49:47.191040+0000","last_unstale":"2024-04-05T02:49:49.202914+0000","last_undegraded":"2024-04-05T02:49:49.202914+0000","last_fullsized":"2024-04-05T02:49:49.202914+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:40:39.022369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-05T02:50:33.934345+0000","last_change":"2024-04-05T02:50:33.934345+0000","last_active":"2024-04-05T02:50:33.934345+0000","last_peered":"2024-04-05T02:50:33.934345+0000","last_clean":"2024-04-05T02:50:33.934345+0000","last_became_active":"2024-04-05T02:50:33.933968+0000","last_became_peered":"2024-04-05T02:50:33.933968+0000","last_unstale":"2024-04-05T02:50:33.934345+0000","last_undegraded":"2024-04-05T02:50:33.934345+0000","last_fullsized":"2024-04-05T02:50:33.934345+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:58:29.769992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931029+0000","last_change":"2024-04-05T02:49:47.191189+0000","last_active":"2024-04-05T02:50:32.931029+0000","last_peered":"2024-04-05T02:50:32.931029+0000","last_clean":"2024-04-05T02:50:32.931029+0000","last_became_active":"2024-04-05T02:49:47.190925+0000","last_became_peered":"2024-04-05T02:49:47.190925+0000","last_unstale":"2024-04-05T02:50:32.931029+0000","last_undegraded":"2024-04-05T02:50:32.931029+0000","last_fullsized":"2024-04-05T02:50:32.931029+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:49:35.432562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:50.422076+0000","last_change":"2024-04-05T02:49:47.208253+0000","last_active":"2024-04-05T02:49:50.422076+0000","last_peered":"2024-04-05T02:49:50.422076+0000","last_clean":"2024-04-05T02:49:50.422076+0000","last_became_active":"2024-04-05T02:49:47.207967+0000","last_became_peered":"2024-04-05T02:49:47.207967+0000","last_unstale":"2024-04-05T02:49:50.422076+0000","last_undegraded":"2024-04-05T02:49:50.422076+0000","last_fullsized":"2024-04-05T02:49:50.422076+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:48:22.609490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":19,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.913798+0000","last_change":"2024-04-05T02:49:47.188521+0000","last_active":"2024-04-05T02:49:58.913798+0000","last_peered":"2024-04-05T02:49:58.913798+0000","last_clean":"2024-04-05T02:49:58.913798+0000","last_became_active":"2024-04-05T02:49:47.188231+0000","last_became_peered":"2024-04-05T02:49:47.188231+0000","last_unstale":"2024-04-05T02:49:58.913798+0000","last_undegraded":"2024-04-05T02:49:58.913798+0000","last_fullsized":"2024-04-05T02:49:58.913798+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:49:13.887883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931533+0000","last_change":"2024-04-05T02:49:48.245097+0000","last_active":"2024-04-05T02:50:32.931533+0000","last_peered":"2024-04-05T02:50:32.931533+0000","last_clean":"2024-04-05T02:50:32.931533+0000","last_became_active":"2024-04-05T02:49:48.244750+0000","last_became_peered":"2024-04-05T02:49:48.244750+0000","last_unstale":"2024-04-05T02:50:32.931533+0000","last_undegraded":"2024-04-05T02:50:32.931533+0000","last_fullsized":"2024-04-05T02:50:32.931533+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:42:16.800839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931166+0000","last_change":"2024-04-05T02:49:47.196780+0000","last_active":"2024-04-05T02:50:32.931166+0000","last_peered":"2024-04-05T02:50:32.931166+0000","last_clean":"2024-04-05T02:50:32.931166+0000","last_became_active":"2024-04-05T02:49:47.196491+0000","last_became_peered":"2024-04-05T02:49:47.196491+0000","last_unstale":"2024-04-05T02:50:32.931166+0000","last_undegraded":"2024-04-05T02:50:32.931166+0000","last_fullsized":"2024-04-05T02:50:32.931166+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:30:05.059541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928592+0000","last_change":"2024-04-05T02:49:48.245084+0000","last_active":"2024-04-05T02:50:32.928592+0000","last_peered":"2024-04-05T02:50:32.928592+0000","last_clean":"2024-04-05T02:50:32.928592+0000","last_became_active":"2024-04-05T02:49:48.244766+0000","last_became_peered":"2024-04-05T02:49:48.244766+0000","last_unstale":"2024-04-05T02:50:32.928592+0000","last_undegraded":"2024-04-05T02:50:32.928592+0000","last_fullsized":"2024-04-05T02:50:32.928592+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:58:40.299428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.930382+0000","last_change":"2024-04-05T02:49:47.189264+0000","last_active":"2024-04-05T02:50:32.930382+0000","last_peered":"2024-04-05T02:50:32.930382+0000","last_clean":"2024-04-05T02:50:32.930382+0000","last_became_active":"2024-04-05T02:49:47.188944+0000","last_became_peered":"2024-04-05T02:49:47.188944+0000","last_unstale":"2024-04-05T02:50:32.930382+0000","last_undegraded":"2024-04-05T02:50:32.930382+0000","last_fullsized":"2024-04-05T02:50:32.930382+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:29:15.760759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928527+0000","last_change":"2024-04-05T02:49:48.245246+0000","last_active":"2024-04-05T02:50:32.928527+0000","last_peered":"2024-04-05T02:50:32.928527+0000","last_clean":"2024-04-05T02:50:32.928527+0000","last_became_active":"2024-04-05T02:49:48.244713+0000","last_became_peered":"2024-04-05T02:49:48.244713+0000","last_unstale":"2024-04-05T02:50:32.928527+0000","last_undegraded":"2024-04-05T02:50:32.928527+0000","last_fullsized":"2024-04-05T02:50:32.928527+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:47:33.980664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203658+0000","last_change":"2024-04-05T02:49:47.198524+0000","last_active":"2024-04-05T02:49:49.203658+0000","last_peered":"2024-04-05T02:49:49.203658+0000","last_clean":"2024-04-05T02:49:49.203658+0000","last_became_active":"2024-04-05T02:49:47.198036+0000","last_became_peered":"2024-04-05T02:49:47.198036+0000","last_unstale":"2024-04-05T02:49:49.203658+0000","last_undegraded":"2024-04-05T02:49:49.203658+0000","last_fullsized":"2024-04-05T02:49:49.203658+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:13:44.697687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.933204+0000","last_change":"2024-04-05T02:49:47.208079+0000","last_active":"2024-04-05T02:50:32.933204+0000","last_peered":"2024-04-05T02:50:32.933204+0000","last_clean":"2024-04-05T02:50:32.933204+0000","last_became_active":"2024-04-05T02:49:47.207741+0000","last_became_peered":"2024-04-05T02:49:47.207741+0000","last_unstale":"2024-04-05T02:50:32.933204+0000","last_undegraded":"2024-04-05T02:50:32.933204+0000","last_fullsized":"2024-04-05T02:50:32.933204+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:05:02.545819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928540+0000","last_change":"2024-04-05T02:49:47.198831+0000","last_active":"2024-04-05T02:50:32.928540+0000","last_peered":"2024-04-05T02:50:32.928540+0000","last_clean":"2024-04-05T02:50:32.928540+0000","last_became_active":"2024-04-05T02:49:47.198570+0000","last_became_peered":"2024-04-05T02:49:47.198570+0000","last_unstale":"2024-04-05T02:50:32.928540+0000","last_undegraded":"2024-04-05T02:50:32.928540+0000","last_fullsized":"2024-04-05T02:50:32.928540+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:49:16.622423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:49.203150+0000","last_change":"2024-04-05T02:49:48.221939+0000","last_active":"2024-04-05T02:49:49.203150+0000","last_peered":"2024-04-05T02:49:49.203150+0000","last_clean":"2024-04-05T02:49:49.203150+0000","last_became_active":"2024-04-05T02:49:48.221603+0000","last_became_peered":"2024-04-05T02:49:48.221603+0000","last_unstale":"2024-04-05T02:49:49.203150+0000","last_undegraded":"2024-04-05T02:49:49.203150+0000","last_fullsized":"2024-04-05T02:49:49.203150+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:50:24.489229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.931420+0000","last_change":"2024-04-05T02:49:47.198624+0000","last_active":"2024-04-05T02:50:32.931420+0000","last_peered":"2024-04-05T02:50:32.931420+0000","last_clean":"2024-04-05T02:50:32.931420+0000","last_became_active":"2024-04-05T02:49:47.198354+0000","last_became_peered":"2024-04-05T02:49:47.198354+0000","last_unstale":"2024-04-05T02:50:32.931420+0000","last_undegraded":"2024-04-05T02:50:32.931420+0000","last_fullsized":"2024-04-05T02:50:32.931420+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:14:26.367910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-04-05T02:50:32.928932+0000","last_change":"2024-04-05T02:49:48.245923+0000","last_active":"2024-04-05T02:50:32.928932+0000","last_peered":"2024-04-05T02:50:32.928932+0000","last_clean":"2024-04-05T02:50:32.928932+0000","last_became_active":"2024-04-05T02:49:48.245629+0000","last_became_peered":"2024-04-05T02:49:48.245629+0000","last_unstale":"2024-04-05T02:50:32.928932+0000","last_undegraded":"2024-04-05T02:50:32.928932+0000","last_fullsized":"2024-04-05T02:50:32.928932+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:39:09.813927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":14,"reported_epoch":69,"state":"active+clean","last_fresh":"2024-04-05T02:49:58.908457+0000","last_change":"2024-04-05T02:49:47.194662+0000","last_active":"2024-04-05T02:49:58.908457+0000","last_peered":"2024-04-05T02:49:58.908457+0000","last_clean":"2024-04-05T02:49:58.908457+0000","last_became_active":"2024-04-05T02:49:47.194047+0000","last_became_peered":"2024-04-05T02:49:47.194047+0000","last_unstale":"2024-04-05T02:49:58.908457+0000","last_undegraded":"2024-04-05T02:49:58.908457+0000","last_fullsized":"2024-04-05T02:49:58.908457+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:00:36.223513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298676,"kb_used_data":1316,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446476,"statfs":{"total":95995035648,"available":95689191424,"internally_reserved":0,"allocated":1347584,"data_stored":908552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.53300000000000003,"15min":0.52800000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.52600000000000002,"15min":0.52400000000000002},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.74099999999999999,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.46700000000000003}]},{"osd":1,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.378,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68799999999999994,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.39400000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.51100000000000001}]},{"osd":2,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.39200000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.73499999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.55000000000000004,"15min":0.55100000000000005},"min":{"1min":0.40899999999999997,"5min":0.371,"15min":0.371},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.55100000000000005}]},{"osd":3,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.51800000000000002,"15min":0.51400000000000001},"min":{"1min":0.42099999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72399999999999998,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.54000000000000004,"15min":0.53600000000000003},"min":{"1min":0.39600000000000002,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.70699999999999996}]},{"osd":4,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.5,"15min":0.5},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.63800000000000001}]},{"osd":5,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.504,"15min":0.50700000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.83699999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.51500000000000001,"15min":0.51000000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.59399999999999997}]},{"osd":6,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.68799999999999994}]}]},{"osd":6,"up_from":47,"seq":201863462955,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298664,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446488,"statfs":{"total":95995035648,"available":95689203712,"internally_reserved":0,"allocated":1335296,"data_stored":908406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.51600000000000001,"15min":0.499},"min":{"1min":0.40500000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.51100000000000001,"15min":0.502},"min":{"1min":0.377,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.45200000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:50:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.52300000000000002,"15min":0.51100000000000001},"min":{"1min":0.35299999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.441},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.51100000000000001,"15min":0.502},"min":{"1min":0.39000000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.51100000000000001}]},{"osd":2,"last update":"Fri Apr 5 02:50:45 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.502,"15min":0.48399999999999999},"min":{"1min":0.40999999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.53700000000000003,"15min":0.52900000000000003},"min":{"1min":0.39700000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.48799999999999999}]},{"osd":3,"last update":"Fri Apr 5 02:50:45 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.40899999999999997,"5min":0.375,"15min":0.375},"max":{"1min":0.72599999999999998,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.52100000000000002,"15min":0.50800000000000001},"min":{"1min":0.42199999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.75,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.53900000000000003}]},{"osd":4,"last update":"Fri Apr 5 02:50:45 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.40500000000000003,"15min":0.38200000000000001},"min":{"1min":0.314,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.42199999999999999,"15min":0.40000000000000002},"min":{"1min":0.374,"5min":0.252,"15min":0.252},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.622}]},{"osd":5,"last update":"Fri Apr 5 02:50:45 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.439,"15min":0.42099999999999999},"min":{"1min":0.318,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.495,"15min":0.46100000000000002},"min":{"1min":0.41899999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.57899999999999996}]},{"osd":7,"last update":"Fri Apr 5 02:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.48199999999999998,"15min":0.47999999999999998},"min":{"1min":0.34300000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.624,"5min":0.751,"15min":0.751},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.51700000000000002,"15min":0.51100000000000001},"min":{"1min":0.40400000000000003,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.71099999999999997,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.61199999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":316151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.371,"15min":0.32800000000000001},"min":{"1min":0.26700000000000002,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.53800000000000003,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.39600000000000002,"15min":0.34899999999999998},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.67800000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.40300000000000002}]},{"osd":2,"last update":"Fri Apr 5 02:50:04 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.376,"15min":0.373},"min":{"1min":0.29199999999999998,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.51300000000000001,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.313},{"interface":"front","average":{"1min":0.41499999999999998,"5min":0.41299999999999998,"15min":0.39500000000000002},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.57899999999999996,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.33200000000000002}]},{"osd":3,"last update":"Fri Apr 5 02:50:29 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.46100000000000002,"15min":0.47899999999999998},"min":{"1min":0.32400000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.5,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.378},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.42599999999999999,"15min":0.44600000000000001},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":0.58599999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"last":0.35299999999999998}]},{"osd":4,"last update":"Fri Apr 5 02:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.61499999999999999,"15min":0.629},"min":{"1min":0.41499999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.88300000000000001,"5min":1.022,"15min":1.022},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58299999999999996,"15min":0.59099999999999997},"min":{"1min":0.40000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.55100000000000005}]},{"osd":5,"last update":"Fri Apr 5 02:50:15 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.56699999999999995,"15min":0.57499999999999996},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60899999999999999,"15min":0.624},"min":{"1min":0.42999999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.85899999999999999,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.497}]},{"osd":6,"last update":"Fri Apr 5 02:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.55000000000000004},"min":{"1min":0.44,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.59099999999999997,"15min":0.58799999999999997},"min":{"1min":0.46999999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.89900000000000002,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.52600000000000002}]},{"osd":7,"last update":"Fri Apr 5 02:50:04 2024","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.40300000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.437},{"interface":"front","average":{"1min":0.629,"5min":0.56799999999999995,"15min":0.55800000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.57799999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298652,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446500,"statfs":{"total":95995035648,"available":95689216000,"internally_reserved":0,"allocated":1327104,"data_stored":907376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Apr 5 02:50:43 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.35199999999999998,"15min":0.32400000000000001},"min":{"1min":0.22,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.501,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.374},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.34899999999999998},"min":{"1min":0.26100000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.52900000000000003,"5min":0.56999999999999995,"15min":0.56999999999999995},"last":0.26800000000000002}]},{"osd":2,"last update":"Fri Apr 5 02:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.41599999999999998,"15min":0.438},"min":{"1min":0.27300000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.52400000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.37,"5min":0.39800000000000002,"15min":0.39900000000000002},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.48899999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Fri Apr 5 02:50:31 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.434,"15min":0.45000000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.51500000000000001,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.42099999999999999,"15min":0.42299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.53600000000000003,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.32900000000000001}]},{"osd":4,"last update":"Fri Apr 5 02:49:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53600000000000003,"15min":0.53200000000000003},"min":{"1min":0.40600000000000003,"5min":0.377,"15min":0.377},"max":{"1min":0.623,"5min":0.875,"15min":0.875},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54700000000000004,"15min":0.53700000000000003},"min":{"1min":0.42199999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.68000000000000005,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Apr 5 02:50:16 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.59599999999999997,"15min":0.59999999999999998},"min":{"1min":0.41899999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79000000000000004,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.58599999999999997,"15min":0.59799999999999998},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79100000000000004,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.71099999999999997}]},{"osd":6,"last update":"Fri Apr 5 02:50:43 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.60899999999999999,"15min":0.627},"min":{"1min":0.44500000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76000000000000001,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61499999999999999,"15min":0.63300000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.82199999999999995,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.64500000000000002}]},{"osd":7,"last update":"Fri Apr 5 02:50:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.45800000000000002,"5min":0.435,"15min":0.435},"max":{"1min":0.752,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.622,"15min":0.628},"min":{"1min":0.48599999999999999,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.76800000000000002,"5min":1.1639999999999999,"15min":1.1639999999999999},"last":0.70299999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.39300000000000002,"15min":0.39100000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.48799999999999999,"5min":0.62,"15min":0.62},"last":0.32300000000000001}]},{"osd":1,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.41199999999999998,"15min":0.39900000000000002},"min":{"1min":0.28199999999999997,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.61899999999999999,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.44900000000000001,"15min":0.42899999999999999},"min":{"1min":0.318,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.52900000000000003,"5min":0.751,"15min":0.751},"last":0.45300000000000001}]},{"osd":3,"last update":"Fri Apr 5 02:50:33 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48999999999999999,"15min":0.501},"min":{"1min":0.34799999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.42999999999999999,"15min":0.45800000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.327,"5min":1.327,"15min":1.327},"last":0.35299999999999998}]},{"osd":4,"last update":"Fri Apr 5 02:49:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.60199999999999998,"15min":0.60399999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.77100000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.59299999999999997,"15min":0.61499999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.77200000000000002,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.60399999999999998}]},{"osd":5,"last update":"Fri Apr 5 02:50:12 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.57099999999999995,"15min":0.58299999999999996},"min":{"1min":0.372,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73399999999999999,"5min":0.745,"15min":0.745},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.47599999999999998,"5min":0.443,"15min":0.443},"max":{"1min":0.83199999999999996,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.59299999999999997}]},{"osd":6,"last update":"Fri Apr 5 02:50:40 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.63700000000000001,"15min":0.63600000000000001},"min":{"1min":0.47899999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.1459999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.59499999999999997,"15min":0.59199999999999997},"min":{"1min":0.45700000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.29,"5min":1.29,"15min":1.29},"last":0.65300000000000002}]},{"osd":7,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.58799999999999997,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.76100000000000001,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.629,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.47499999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.71799999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298064,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447088,"statfs":{"total":95995035648,"available":95689818112,"internally_reserved":0,"allocated":720896,"data_stored":315918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.40100000000000002,"15min":0.39400000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.497,"5min":1.252,"15min":1.252},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.34200000000000003,"5min":0.40400000000000003,"15min":0.40999999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.48999999999999999,"5min":0.63,"15min":0.63},"last":0.40600000000000003}]},{"osd":1,"last update":"Fri Apr 5 02:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.41199999999999998,"15min":0.42199999999999999},"min":{"1min":0.248,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.46899999999999997,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.374,"5min":0.433,"15min":0.432},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.46700000000000003,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.39300000000000002}]},{"osd":2,"last update":"Fri Apr 5 02:50:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"min":{"1min":0.313,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.55100000000000005,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.437,"15min":0.45100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.44400000000000001,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.43099999999999999}]},{"osd":4,"last update":"Fri Apr 5 02:49:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58999999999999997,"15min":0.59299999999999997},"min":{"1min":0.41799999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.80100000000000005,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58999999999999997,"15min":0.58199999999999996},"min":{"1min":0.44400000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.98699999999999999,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.46600000000000003}]},{"osd":5,"last update":"Fri Apr 5 02:50:12 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.61199999999999999,"15min":0.60499999999999998},"min":{"1min":0.45700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.80900000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.747,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.625}]},{"osd":6,"last update":"Fri Apr 5 02:50:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.60499999999999998,"15min":0.60999999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.69799999999999995,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64400000000000002,"15min":0.64000000000000001},"min":{"1min":0.45900000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.83099999999999996,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Apr 5 02:50:05 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.625,"15min":0.629},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.78400000000000003,"5min":0.88,"15min":0.88},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.65500000000000003,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.84799999999999998,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.53200000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986228,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298088,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447064,"statfs":{"total":95995035648,"available":95689793536,"internally_reserved":0,"allocated":749568,"data_stored":318114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.49299999999999999,"15min":0.5},"min":{"1min":0.25700000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.60799999999999998,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.505,"15min":0.495},"min":{"1min":0.35999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":0.66000000000000003,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.49299999999999999}]},{"osd":1,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.33400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":0.58999999999999997,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.30099999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.59499999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.55300000000000005}]},{"osd":2,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.502,"15min":0.496},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.437},{"interface":"front","average":{"1min":0.498,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.70199999999999996,"5min":0.747,"15min":0.747},"last":0.56899999999999995}]},{"osd":3,"last update":"Fri Apr 5 02:49:53 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54400000000000004,"15min":0.53900000000000003},"min":{"1min":0.375,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.69499999999999995,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.54800000000000004,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.37,"15min":0.37},"max":{"1min":0.66100000000000003,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.38200000000000001}]},{"osd":5,"last update":"Fri Apr 5 02:50:14 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.437,"15min":0.42299999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.68799999999999994,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.432,"15min":0.41199999999999998},"min":{"1min":0.32700000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.67500000000000004,"5min":0.754,"15min":0.754},"last":0.40300000000000002}]},{"osd":6,"last update":"Fri Apr 5 02:50:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.47999999999999998,"15min":0.46100000000000002},"min":{"1min":0.35499999999999998,"5min":0.254,"15min":0.254},"max":{"1min":0.72499999999999998,"5min":0.755,"15min":0.755},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.495,"15min":0.47299999999999998},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.59099999999999997}]},{"osd":7,"last update":"Fri Apr 5 02:50:03 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53400000000000003,"15min":0.53000000000000003},"min":{"1min":0.437,"5min":0.316,"15min":0.316},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.54100000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":708,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":724992,"data_stored":316468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.55800000000000005},"min":{"1min":0.41299999999999998,"5min":0.25,"15min":0.25},"max":{"1min":0.66300000000000003,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.505,"5min":0.51900000000000002,"15min":0.53400000000000003},"min":{"1min":0.35699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.753,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.504}]},{"osd":1,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.73099999999999998,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.38300000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.69499999999999995,"5min":0.745,"15min":0.745},"last":0.40100000000000002}]},{"osd":2,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.52300000000000002,"15min":0.53100000000000003},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.67400000000000004,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55900000000000005,"15min":0.57399999999999995},"min":{"1min":0.41599999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.71199999999999997,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56100000000000005}]},{"osd":3,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.70899999999999996,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55400000000000005,"15min":0.55800000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.72999999999999998,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54800000000000004}]},{"osd":4,"last update":"Fri Apr 5 02:50:17 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.44700000000000001,"15min":0.434},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.751,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.44800000000000001,"15min":0.433},"min":{"1min":0.29299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66800000000000004,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53600000000000003}]},{"osd":6,"last update":"Fri Apr 5 02:50:42 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.501,"15min":0.48899999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.501,"15min":0.49399999999999999},"min":{"1min":0.33100000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.69899999999999995,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.59099999999999997}]},{"osd":7,"last update":"Fri Apr 5 02:50:03 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.374,"5min":0.373,"15min":0.373},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.626},{"interface":"front","average":{"1min":0.502,"5min":0.54500000000000004,"15min":0.55200000000000005},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.69199999999999995,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.57699999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T02:50:49.942 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-05T02:50:49.943 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-05T02:50:49.943 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-05T02:50:49.943 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T02:50:50.203 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:50.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:49 smithi150 ceph-mon[29558]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:50.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:49 smithi150 ceph-mon[29558]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:50:50.217 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:49 smithi150 ceph-mon[41818]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:50.217 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:49 smithi150 ceph-mon[41818]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:50:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:49 smithi196 ceph-mon[33169]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:49 smithi196 ceph-mon[33169]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T02:50:50.472 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:50] "GET /metrics HTTP/1.1" 200 239345 "" "Prometheus/2.43.0" 2024-04-05T02:50:52.023 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T02:50:52.023 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-05T02:50:52.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:51 smithi150 ceph-mon[29558]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:52.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:51 smithi150 ceph-mon[41818]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:51 smithi196 ceph-mon[33169]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:52.819 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-05T02:50:52.819 INFO:teuthology.run_tasks:Running task print... 2024-04-05T02:50:52.829 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-05T02:50:52.829 INFO:teuthology.run_tasks:Running task parallel... 2024-04-05T02:50:52.862 INFO:teuthology.task.parallel:starting parallel... 2024-04-05T02:50:52.862 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-05T02:50:52.863 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-04-05T02:50:52.863 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-05T02:50:52.865 INFO:tasks.radosbench:Beginning radosbench... 2024-04-05T02:50:52.865 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-04-05T02:50:52.865 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool create unique_pool_0 16 2024-04-05T02:50:52.865 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-05T02:50:52.866 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-05T02:50:52.866 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi150.front.sepia.ceph.com 2024-04-05T02:50:52.866 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-05T02:50:52.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:50:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:50:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:50:53.152 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:53.156 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:53.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:52 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3359232984' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T02:50:53.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:52 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3359232984' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T02:50:53.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:52 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3359232984' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T02:50:54.182 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:53 smithi150 ceph-mon[29558]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:54.182 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:53 smithi150 ceph-mon[41818]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:53 smithi196 ceph-mon[33169]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:56.220 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:55 smithi150 ceph-mon[29558]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:56.221 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:55 smithi150 ceph-mon[41818]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:56.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:55 smithi196 ceph-mon[33169]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:56.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:50:56] ENGINE Bus STOPPING 2024-04-05T02:50:56.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:50:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:50:56.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:50:56] ENGINE Bus STOPPED 2024-04-05T02:50:56.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:50:56] ENGINE Bus STARTING 2024-04-05T02:50:56.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:50:56] ENGINE Serving on http://:::9283 2024-04-05T02:50:56.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:50:56] ENGINE Bus STARTED 2024-04-05T02:50:57.001 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_0' created 2024-04-05T02:50:57.133 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/242038124' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-05T02:50:57.133 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-05T02:50:57.133 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' 2024-04-05T02:50:57.133 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:57.134 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/242038124' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-05T02:50:57.134 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-05T02:50:57.134 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' 2024-04-05T02:50:57.134 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:56 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:57.158 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-05T02:50:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:56 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/242038124' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-05T02:50:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:56 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-04-05T02:50:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:56 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' 2024-04-05T02:50:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:56 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:50:57.677 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-04-05T02:50:57.746 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:58.288 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:58 smithi196 ceph-mon[33169]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:58.288 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:58 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-05T02:50:58.288 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:58 smithi196 ceph-mon[33169]: osdmap e72: 8 total, 8 up, 8 in 2024-04-05T02:50:58.289 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:58 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:58.289 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:58 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:58.289 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:58 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:58.290 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[29558]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[29558]: osdmap e72: 8 total, 8 up, 8 in 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[41818]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:57 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:58 smithi150 ceph-mon[41818]: osdmap e72: 8 total, 8 up, 8 in 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:58 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:58 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:50:58.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:58 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:50:58.511 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:50:59.264 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:50:59 smithi150 ceph-mon[41818]: osdmap e73: 8 total, 8 up, 8 in 2024-04-05T02:50:59.264 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:50:59 smithi150 ceph-mon[29558]: osdmap e73: 8 total, 8 up, 8 in 2024-04-05T02:50:59.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:50:59 smithi196 ceph-mon[33169]: osdmap e73: 8 total, 8 up, 8 in 2024-04-05T02:51:00.321 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-mon[29558]: pgmap v89: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:00.321 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-mon[29558]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:51:00.321 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-mon[29558]: osdmap e74: 8 total, 8 up, 8 in 2024-04-05T02:51:00.322 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-mon[41818]: pgmap v89: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:00.322 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-mon[41818]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:51:00.322 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-mon[41818]: osdmap e74: 8 total, 8 up, 8 in 2024-04-05T02:51:00.322 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:51:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:00] "GET /metrics HTTP/1.1" 200 239345 "" "Prometheus/2.43.0" 2024-04-05T02:51:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:00 smithi196 ceph-mon[33169]: pgmap v89: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:00 smithi196 ceph-mon[33169]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:51:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:00 smithi196 ceph-mon[33169]: osdmap e74: 8 total, 8 up, 8 in 2024-04-05T02:51:01.639 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-05T02:51:01.905 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:51:02.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:02 smithi150 ceph-mon[29558]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:02.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:02 smithi150 ceph-mon[41818]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:02 smithi196 ceph-mon[33169]: pgmap v91: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:51:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:51:03.025 INFO:teuthology.orchestra.run.smithi150.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3099279785' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3099279785' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:03.104 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:03 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:03.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:03 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3099279785' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:51:03.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:03 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:51:03.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:03 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:03.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:03 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:04 smithi196 ceph-mon[33169]: pgmap v92: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:04 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:51:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:04 smithi196 ceph-mon[33169]: osdmap e75: 8 total, 8 up, 8 in 2024-04-05T02:51:04.450 INFO:teuthology.orchestra.run.smithi150.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:04.450 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:04 smithi150 ceph-mon[29558]: pgmap v92: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:04.450 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:04 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:51:04.450 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:04 smithi150 ceph-mon[29558]: osdmap e75: 8 total, 8 up, 8 in 2024-04-05T02:51:04.451 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:04 smithi150 ceph-mon[41818]: pgmap v92: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:04.451 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:04 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:51:04.451 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:04 smithi150 ceph-mon[41818]: osdmap e75: 8 total, 8 up, 8 in 2024-04-05T02:51:04.784 DEBUG:teuthology.orchestra.run.smithi150:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-04-05T02:51:04.786 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-05T02:51:04.786 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-05T02:51:04.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:hints = 1 2024-04-05T02:51:04.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-05T02:51:04.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object prefix: benchmark_data_smithi150_87008 2024-04-05T02:51:04.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:51:04.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 0 0 0 0 0 0 - 0 2024-04-05T02:51:05.163 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T02:51:05.662 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:51:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:51:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[29558]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:51:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:51:05.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:51:05.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:05.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:05.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:05 smithi150 ceph-mon[41818]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:05.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 1 16 4031 4015 250.924 250.938 0.00251693 0.00397472 2024-04-05T02:51:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: from='client.14802 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:51:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:0028525c15e19309c20289c609e4fb9d0b12dd06 2024-04-05T02:51:05.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:05 smithi196 ceph-mon[33169]: pgmap v94: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:06 smithi150 ceph-mon[41818]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:51:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:06 smithi150 ceph-mon[41818]: Cluster is now healthy 2024-04-05T02:51:06.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:06 smithi150 ceph-mon[29558]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:51:06.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:06 smithi150 ceph-mon[29558]: Cluster is now healthy 2024-04-05T02:51:06.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 2 16 7596 7580 236.859 222.812 0.00367899 0.00421416 2024-04-05T02:51:06.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:06 smithi196 ceph-mon[33169]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:51:06.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:06 smithi196 ceph-mon[33169]: Cluster is now healthy 2024-04-05T02:51:07.595 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:07 smithi150 ceph-mon[41818]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:07.595 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:07 smithi150 ceph-mon[29558]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:07.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 3 16 10716 10700 222.9 195 0.00595141 0.00447733 2024-04-05T02:51:07.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:07 smithi196 ceph-mon[33169]: pgmap v95: 89 pgs: 89 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:51:08.214 INFO:teuthology.orchestra.run.smithi150.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T02:51:08.214 INFO:teuthology.orchestra.run.smithi150.stdout:alertmanager.a smithi150 *:9093,9094 running (85s) 28s ago 2m 16.7M - 0.25.0 c8568f914cd2 05286344493f 2024-04-05T02:51:08.214 INFO:teuthology.orchestra.run.smithi150.stdout:grafana.a smithi196 *:3000 running (2m) 57s ago 2m 82.2M - 9.4.7 954c08fa6188 75718600d444 2024-04-05T02:51:08.214 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi150.jnrwgs smithi150 running (70s) 28s ago 69s 12.4M - 17.2.7-805-g6fda41ff a49113eacc58 cfa13d4de7a1 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi196.eohomm smithi196 running (67s) 57s ago 67s 14.4M - 17.2.7-805-g6fda41ff a49113eacc58 e3ce4d99c3da 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.x smithi196 *:8443 running (6m) 57s ago 6m 431M - 17.2.7-805-g6fda41ff a49113eacc58 5dcef717d454 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.y smithi150 *:9283 running (8m) 28s ago 8m 475M - 17.2.7-805-g6fda41ff a49113eacc58 be1d8eb95a04 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:mon.a smithi150 running (8m) 28s ago 8m 47.9M 2048M 17.2.7-805-g6fda41ff a49113eacc58 ff4c949de5c0 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:mon.b smithi196 running (6m) 57s ago 6m 35.5M 2048M 17.2.7-805-g6fda41ff a49113eacc58 6680dc817949 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:mon.c smithi150 running (6m) 28s ago 6m 38.3M 2048M 17.2.7-805-g6fda41ff a49113eacc58 826b3558df5d 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.a smithi150 *:9100 running (113s) 28s ago 111s 7763k - 1.5.0 0da6a335fe13 ebab866c2b05 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.b smithi196 *:9100 running (108s) 57s ago 107s 7884k - 1.5.0 0da6a335fe13 1d7f72da9f2d 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.0 smithi150 running (6m) 28s ago 6m 44.0M 4016M 17.2.7-805-g6fda41ff a49113eacc58 a6b8a0140540 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.1 smithi150 running (5m) 28s ago 5m 41.5M 4016M 17.2.7-805-g6fda41ff a49113eacc58 2c503b0a41b4 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.2 smithi150 running (5m) 28s ago 5m 40.5M 4016M 17.2.7-805-g6fda41ff a49113eacc58 fee7e3893826 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.3 smithi150 running (4m) 28s ago 4m 40.6M 4016M 17.2.7-805-g6fda41ff a49113eacc58 5d887a680c09 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.4 smithi196 running (4m) 57s ago 4m 40.5M 4016M 17.2.7-805-g6fda41ff a49113eacc58 b21803209415 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.5 smithi196 running (4m) 57s ago 4m 39.7M 4016M 17.2.7-805-g6fda41ff a49113eacc58 5415e890a870 2024-04-05T02:51:08.215 INFO:teuthology.orchestra.run.smithi150.stdout:osd.6 smithi196 running (3m) 57s ago 3m 41.7M 4016M 17.2.7-805-g6fda41ff a49113eacc58 85a4b25b62cf 2024-04-05T02:51:08.216 INFO:teuthology.orchestra.run.smithi150.stdout:osd.7 smithi196 running (3m) 57s ago 3m 41.1M 4016M 17.2.7-805-g6fda41ff a49113eacc58 ed60a8dad4f7 2024-04-05T02:51:08.216 INFO:teuthology.orchestra.run.smithi150.stdout:prometheus.a smithi196 *:9095 running (80s) 57s ago 2m 33.1M - 2.43.0 a07b618ecd1d 036e6d335f1a 2024-04-05T02:51:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:08 smithi150 ceph-mon[41818]: from='client.24640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:51:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:08 smithi150 ceph-mon[29558]: from='client.24640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:51:08.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 4 16 13516 13500 210.923 175 0.00396345 0.00473442 2024-04-05T02:51:08.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:08 smithi196 ceph-mon[33169]: from='client.24640 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:51:09.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:09 smithi150 ceph-mon[41818]: pgmap v96: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-04-05T02:51:09.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:09 smithi150 ceph-mon[29558]: pgmap v96: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-04-05T02:51:09.777 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-05T02:51:09.830 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 5 16 16796 16780 209.735 205 0.00420551 0.00476274 2024-04-05T02:51:09.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:09 smithi196 ceph-mon[33169]: pgmap v96: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-04-05T02:51:10.245 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:51:10.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:51:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:10] "GET /metrics HTTP/1.1" 200 241639 "" "Prometheus/2.43.0" 2024-04-05T02:51:10.831 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 6 16 19652 19636 204.525 178.5 0.00681913 0.00488371 2024-04-05T02:51:11.831 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 7 16 22546 22530 201.144 180.875 0.00585396 0.0049656 2024-04-05T02:51:11.936 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:11 smithi150 ceph-mon[29558]: pgmap v97: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-05T02:51:11.937 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:11 smithi150 ceph-mon[41818]: pgmap v97: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-05T02:51:12.298 INFO:teuthology.orchestra.run.smithi150.stdout:wait for minority of mons to upgrade 2024-04-05T02:51:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:11 smithi196 ceph-mon[33169]: pgmap v97: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-05T02:51:12.831 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 8 16 25251 25235 197.132 169.062 0.00585963 0.00506641 2024-04-05T02:51:12.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:51:12 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:51:13.187 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-05T02:51:13.645 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:51:13.832 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 9 16 27691 27675 192.164 152.5 0.00595599 0.0051988 2024-04-05T02:51:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:13 smithi150 ceph-mon[41818]: pgmap v98: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-05T02:51:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:13 smithi150 ceph-mon[29558]: pgmap v98: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-05T02:51:14.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:13 smithi196 ceph-mon[33169]: pgmap v98: 89 pgs: 89 active+clean; 847 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2024-04-05T02:51:14.831 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 10 16 30122 30106 188.14 151.938 0.00694294 0.00530963 2024-04-05T02:51:15.832 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 11 16 32286 32270 183.331 135.25 0.00790112 0.00544983 2024-04-05T02:51:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:15 smithi150 ceph-mon[29558]: pgmap v99: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-04-05T02:51:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:15 smithi150 ceph-mon[41818]: pgmap v99: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-04-05T02:51:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:15 smithi196 ceph-mon[33169]: pgmap v99: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-04-05T02:51:16.832 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 12 16 34406 34390 179.094 132.5 0.00323937 0.00557765 2024-04-05T02:51:17.833 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 13 16 36215 36199 173.995 113.062 0.00438088 0.00574186 2024-04-05T02:51:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:17 smithi150 ceph-mon[41818]: pgmap v100: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-04-05T02:51:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:17 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:17 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:17 smithi150 ceph-mon[29558]: pgmap v100: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-04-05T02:51:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:17 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:17 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:18.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:17 smithi196 ceph-mon[33169]: pgmap v100: 89 pgs: 89 active+clean; 1.7 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-04-05T02:51:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:17 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:17 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:18.833 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 14 16 38195 38179 170.405 123.75 0.00557584 0.00586363 2024-04-05T02:51:19.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2147872006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:19.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2147872006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:19.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2147872006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:19.833 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 15 16 40359 40343 168.061 135.25 0.0077127 0.00594625 2024-04-05T02:51:20.218 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:19 smithi150 ceph-mon[41818]: pgmap v101: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 199 MiB/s wr, 3.18k op/s 2024-04-05T02:51:20.219 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:19 smithi150 ceph-mon[29558]: pgmap v101: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 199 MiB/s wr, 3.18k op/s 2024-04-05T02:51:20.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:19 smithi196 ceph-mon[33169]: pgmap v101: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 199 MiB/s wr, 3.18k op/s 2024-04-05T02:51:20.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:51:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:20] "GET /metrics HTTP/1.1" 200 242593 "" "Prometheus/2.43.0" 2024-04-05T02:51:20.834 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 16 16 42464 42448 165.779 131.562 0.00733517 0.00602771 2024-04-05T02:51:21.833 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 17 16 44378 44362 163.064 119.625 0.00473023 0.00612806 2024-04-05T02:51:22.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:21 smithi150 ceph-mon[29558]: pgmap v102: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-05T02:51:22.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:21 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1024242681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:22.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:21 smithi150 ceph-mon[41818]: pgmap v102: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-05T02:51:22.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:21 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1024242681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:22.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:21 smithi196 ceph-mon[33169]: pgmap v102: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-05T02:51:22.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1024242681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:22.833 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 18 16 46387 46371 160.98 125.562 0.00565475 0.00620769 2024-04-05T02:51:22.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:51:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:51:23.834 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 19 16 48239 48223 158.6 115.75 0.0141228 0.00629971 2024-04-05T02:51:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:23 smithi150 ceph-mon[41818]: pgmap v103: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-05T02:51:24.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:23 smithi150 ceph-mon[29558]: pgmap v103: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-05T02:51:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:23 smithi196 ceph-mon[33169]: pgmap v103: 89 pgs: 89 active+clean; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.06k op/s 2024-04-05T02:51:24.834 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:51:24.835567+0000 min lat: 0.00110037 max lat: 0.0864146 avg lat: 0.00638707 2024-04-05T02:51:24.834 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:51:24.834 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 20 16 50094 50078 156.464 115.938 0.00921473 0.00638707 2024-04-05T02:51:25.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:24 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1378004649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:24 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1378004649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:25.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:24 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1378004649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:25.834 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 21 16 51927 51911 154.468 114.562 0.00488502 0.00646869 2024-04-05T02:51:26.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:25 smithi150 ceph-mon[29558]: pgmap v104: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2024-04-05T02:51:26.232 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:25 smithi150 ceph-mon[41818]: pgmap v104: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2024-04-05T02:51:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:25 smithi196 ceph-mon[33169]: pgmap v104: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 181 MiB/s wr, 2.89k op/s 2024-04-05T02:51:26.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 22 16 53590 53574 152.162 103.938 0.0058012 0.00656702 2024-04-05T02:51:27.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:26 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/776650118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:26 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/776650118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:27.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:26 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/776650118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:27.835 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 23 16 55327 55311 150.267 108.562 0.00941355 0.00664987 2024-04-05T02:51:28.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:27 smithi150 ceph-mon[29558]: pgmap v105: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-05T02:51:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:27 smithi150 ceph-mon[41818]: pgmap v105: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-05T02:51:28.287 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:27 smithi196 ceph-mon[33169]: pgmap v105: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-04-05T02:51:28.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 24 16 57064 57048 148.529 108.562 0.0188994 0.00672757 2024-04-05T02:51:29.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 25 16 58541 58525 146.28 92.3125 0.00471822 0.00683167 2024-04-05T02:51:30.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:29 smithi196 ceph-mon[33169]: pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s 2024-04-05T02:51:30.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:29 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3876497600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:30.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:29 smithi150 ceph-mon[29558]: pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s 2024-04-05T02:51:30.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:29 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3876497600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:30.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:51:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:30] "GET /metrics HTTP/1.1" 200 242593 "" "Prometheus/2.43.0" 2024-04-05T02:51:30.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:29 smithi150 ceph-mon[41818]: pgmap v106: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 153 MiB/s wr, 2.45k op/s 2024-04-05T02:51:30.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3876497600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:30.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 26 16 60133 60117 144.481 99.5 0.00419215 0.00691604 2024-04-05T02:51:31.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 27 16 61640 61624 142.618 94.1875 0.00899744 0.00700681 2024-04-05T02:51:32.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:31 smithi196 ceph-mon[33169]: pgmap v107: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T02:51:32.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:31 smithi150 ceph-mon[29558]: pgmap v107: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T02:51:32.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:31 smithi150 ceph-mon[41818]: pgmap v107: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T02:51:32.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 28 16 63079 63063 140.736 89.9375 0.0138939 0.00710061 2024-04-05T02:51:32.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:51:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:51:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:33 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3713291030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:33 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:33.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:33 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3713291030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:33.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:33 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:33.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:33 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3713291030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:33.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:33 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:33.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 29 16 64568 64552 139.092 93.0625 0.00662017 0.00718442 2024-04-05T02:51:34.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:34 smithi196 ceph-mon[33169]: pgmap v108: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T02:51:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:34 smithi150 ceph-mon[29558]: pgmap v108: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T02:51:34.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:34 smithi150 ceph-mon[41818]: pgmap v108: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T02:51:34.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 30 16 65977 65961 137.391 88.0625 0.00551637 0.00727328 2024-04-05T02:51:35.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 31 16 67374 67358 135.776 87.3125 0.0120525 0.00735971 2024-04-05T02:51:36.279 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:36 smithi150 ceph-mon[29558]: pgmap v109: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-05T02:51:36.279 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:36 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3211389069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:36.280 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:36 smithi150 ceph-mon[41818]: pgmap v109: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-05T02:51:36.280 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:36 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3211389069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:36 smithi196 ceph-mon[33169]: pgmap v109: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2024-04-05T02:51:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:36 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3211389069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:36.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 32 16 68729 68713 134.179 84.6875 0.00659985 0.00744817 2024-04-05T02:51:37.836 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 33 16 69882 69866 132.297 72.0625 0.00446006 0.00755254 2024-04-05T02:51:38.033 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:38 smithi150 ceph-mon[41818]: pgmap v110: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-05T02:51:38.033 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:38 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2733464338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:38.034 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:38 smithi150 ceph-mon[29558]: pgmap v110: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-05T02:51:38.034 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:38 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2733464338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:38.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:38 smithi196 ceph-mon[33169]: pgmap v110: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-04-05T02:51:38.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:38 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2733464338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:38.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 34 16 71047 71031 130.548 72.8125 0.00777459 0.00765479 2024-04-05T02:51:39.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 35 16 72339 72323 129.125 80.75 0.00516011 0.00773897 2024-04-05T02:51:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:40 smithi196 ceph-mon[33169]: pgmap v111: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-04-05T02:51:40.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:40 smithi150 ceph-mon[41818]: pgmap v111: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-04-05T02:51:40.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:51:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:40] "GET /metrics HTTP/1.1" 200 242755 "" "Prometheus/2.43.0" 2024-04-05T02:51:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:40 smithi150 ceph-mon[29558]: pgmap v111: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2024-04-05T02:51:40.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 36 16 73469 73453 127.5 70.625 0.0150215 0.00783783 2024-04-05T02:51:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:41 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/417050796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:41 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/417050796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:41 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/417050796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:41.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 37 16 74830 74814 126.353 85.0625 0.00765734 0.00790931 2024-04-05T02:51:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:42 smithi196 ceph-mon[33169]: pgmap v112: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-05T02:51:42.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:42 smithi150 ceph-mon[41818]: pgmap v112: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-05T02:51:42.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:42 smithi150 ceph-mon[29558]: pgmap v112: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-05T02:51:42.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 38 16 76086 76070 125.093 78.5 0.0154497 0.00798801 2024-04-05T02:51:42.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:51:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:51:43.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 39 16 77326 77310 123.873 77.5 0.00716476 0.00806827 2024-04-05T02:51:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:44 smithi196 ceph-mon[33169]: pgmap v113: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-05T02:51:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:44 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2733569625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:44.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:44 smithi150 ceph-mon[29558]: pgmap v113: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-05T02:51:44.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:44 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2733569625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:44 smithi150 ceph-mon[41818]: pgmap v113: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-04-05T02:51:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:44 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2733569625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:44.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:51:44.838489+0000 min lat: 0.00110037 max lat: 0.0864146 avg lat: 0.00812295 2024-04-05T02:51:44.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:51:44.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 40 16 78775 78759 123.04 90.5625 0.0158206 0.00812295 2024-04-05T02:51:45.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 41 16 79892 79876 121.742 69.8125 0.00587586 0.00820924 2024-04-05T02:51:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:46 smithi196 ceph-mon[33169]: pgmap v114: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-04-05T02:51:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:46 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/308422491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:46.475 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:46 smithi150 ceph-mon[29558]: pgmap v114: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-04-05T02:51:46.475 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:46 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/308422491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:46.475 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:46 smithi150 ceph-mon[41818]: pgmap v114: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-04-05T02:51:46.475 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:46 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/308422491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:46.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 42 16 81226 81210 120.828 83.375 0.0166716 0.00827148 2024-04-05T02:51:47.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 43 16 82588 82572 119.998 85.125 0.00420801 0.00832758 2024-04-05T02:51:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:48 smithi196 ceph-mon[33169]: pgmap v115: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-04-05T02:51:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:48 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:51:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:48 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:48 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:48.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[41818]: pgmap v115: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-04-05T02:51:48.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:48.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[29558]: pgmap v115: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:51:48.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:48 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:51:48.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 44 16 83838 83822 119.046 78.125 0.0179051 0.00839553 2024-04-05T02:51:49.384 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[29558]: Upgrade: Target is version 19.0.0-2013-g0028525c (unknown) 2024-04-05T02:51:49.384 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[29558]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca'] 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[29558]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[29558]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca on smithi196 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2296110943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[41818]: Upgrade: Target is version 19.0.0-2013-g0028525c (unknown) 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[41818]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca'] 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[41818]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[41818]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca on smithi196 2024-04-05T02:51:49.385 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:49 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2296110943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:49.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:49 smithi196 ceph-mon[33169]: Upgrade: Target is version 19.0.0-2013-g0028525c (unknown) 2024-04-05T02:51:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:49 smithi196 ceph-mon[33169]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, digests ['quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca'] 2024-04-05T02:51:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:49 smithi196 ceph-mon[33169]: Upgrade: Need to upgrade myself (mgr.y) 2024-04-05T02:51:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:49 smithi196 ceph-mon[33169]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca on smithi196 2024-04-05T02:51:49.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:49 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2296110943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:49.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 45 16 85020 85004 118.042 73.875 0.00513034 0.0084669 2024-04-05T02:51:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:50 smithi196 ceph-mon[33169]: pgmap v116: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-05T02:51:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:50 smithi150 ceph-mon[41818]: pgmap v116: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-05T02:51:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:50 smithi150 ceph-mon[29558]: pgmap v116: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-05T02:51:50.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:51:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:50] "GET /metrics HTTP/1.1" 200 242801 "" "Prometheus/2.43.0" 2024-04-05T02:51:50.837 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 46 16 86185 86169 117.059 72.8125 0.0161032 0.0085379 2024-04-05T02:51:51.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 47 16 87362 87346 116.133 73.5625 0.0111788 0.0086055 2024-04-05T02:51:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:52 smithi196 ceph-mon[33169]: pgmap v117: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:51:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:52 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/665624416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:52 smithi150 ceph-mon[29558]: pgmap v117: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:51:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:52 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/665624416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:52.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:52 smithi150 ceph-mon[41818]: pgmap v117: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:51:52.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:52 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/665624416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:52.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 48 16 88533 88517 115.239 73.1875 0.00357576 0.00867265 2024-04-05T02:51:52.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:51:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:51:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:51:53.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 49 15 89639 89624 114.299 69.1875 0.0149948 0.00874455 2024-04-05T02:51:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:54 smithi196 ceph-mon[33169]: pgmap v118: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:51:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:54 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3686083478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:54 smithi150 ceph-mon[41818]: pgmap v118: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:51:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:54 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3686083478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:54.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:54 smithi150 ceph-mon[29558]: pgmap v118: 89 pgs: 89 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:51:54.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:54 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3686083478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:54.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 50 16 90758 90742 113.41 69.875 0.0123449 0.00881248 2024-04-05T02:51:55.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 51 15 91868 91853 112.548 69.4375 0.017118 0.00888022 2024-04-05T02:51:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:56 smithi196 ceph-mon[33169]: pgmap v119: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-05T02:51:56.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:56 smithi150 ceph-mon[29558]: pgmap v119: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-05T02:51:56.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:56 smithi150 ceph-mon[41818]: pgmap v119: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-05T02:51:56.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 52 16 92957 92941 111.691 68 0.0303409 0.00894815 2024-04-05T02:51:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:57 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/842275965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:57.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:57 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/842275965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:57.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:57 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/842275965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:51:57.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 53 16 93953 93937 110.758 62.25 0.0170208 0.00902367 2024-04-05T02:51:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:51:58 smithi196 ceph-mon[33169]: pgmap v120: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T02:51:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:51:58 smithi150 ceph-mon[29558]: pgmap v120: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T02:51:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:51:58 smithi150 ceph-mon[41818]: pgmap v120: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T02:51:58.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 54 16 95043 95027 109.969 68.125 0.0262678 0.00908806 2024-04-05T02:51:59.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 55 16 96094 96078 109.164 65.6875 0.0122472 0.00915562 2024-04-05T02:52:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:00 smithi196 ceph-mon[33169]: pgmap v121: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-05T02:52:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:00 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1194867723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:00.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:00 smithi150 ceph-mon[29558]: pgmap v121: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-05T02:52:00.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:00 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1194867723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:00.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:00] "GET /metrics HTTP/1.1" 200 242801 "" "Prometheus/2.43.0" 2024-04-05T02:52:00.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:00 smithi150 ceph-mon[41818]: pgmap v121: 89 pgs: 89 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-05T02:52:00.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:00 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1194867723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:00.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 56 16 97176 97160 108.422 67.625 0.00788737 0.00921833 2024-04-05T02:52:01.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 57 16 98248 98232 107.695 67 0.0135632 0.00927925 2024-04-05T02:52:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:02 smithi196 ceph-mon[33169]: pgmap v122: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-05T02:52:02.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:02 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3465862807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:02 smithi150 ceph-mon[29558]: pgmap v122: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-05T02:52:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:02 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3465862807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:02.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:02 smithi150 ceph-mon[41818]: pgmap v122: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-05T02:52:02.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:02 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3465862807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:02.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 58 16 99275 99259 106.945 64.1875 0.0120219 0.00934561 2024-04-05T02:52:02.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:52:03.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:03 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:03.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:03 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:03.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:03 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:03.838 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 59 16 100327 100311 106.247 65.75 0.013731 0.00940671 2024-04-05T02:52:04.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:04 smithi196 ceph-mon[33169]: pgmap v123: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2024-04-05T02:52:04.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:04 smithi150 ceph-mon[29558]: pgmap v123: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2024-04-05T02:52:04.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:04 smithi150 ceph-mon[41818]: pgmap v123: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2024-04-05T02:52:04.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:52:04.840191+0000 min lat: 0.00110037 max lat: 0.111643 avg lat: 0.00947025 2024-04-05T02:52:04.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:52:04.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 60 16 101338 101322 105.529 63.1875 0.019081 0.00947025 2024-04-05T02:52:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:05 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2975235961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:05.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:05 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2975235961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:05.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:05 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2975235961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:05.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 61 16 102348 102332 104.834 63.125 0.0133936 0.00953415 2024-04-05T02:52:06.378 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:06 smithi150 ceph-mon[29558]: pgmap v124: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-04-05T02:52:06.378 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:06 smithi150 ceph-mon[41818]: pgmap v124: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-04-05T02:52:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:06 smithi196 ceph-mon[33169]: pgmap v124: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-04-05T02:52:06.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 62 16 103270 103254 104.072 57.625 0.0233747 0.00960376 2024-04-05T02:52:07.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 63 16 104162 104146 103.305 55.75 0.00821597 0.00967564 2024-04-05T02:52:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:08 smithi196 ceph-mon[33169]: pgmap v125: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2024-04-05T02:52:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:08 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/101110442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:08.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:08 smithi150 ceph-mon[29558]: pgmap v125: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2024-04-05T02:52:08.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:08 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/101110442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:08.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:08 smithi150 ceph-mon[41818]: pgmap v125: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 891 op/s 2024-04-05T02:52:08.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:08 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/101110442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:08.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 64 16 105007 104991 102.516 52.8125 0.0146065 0.00974963 2024-04-05T02:52:09.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 65 16 105902 105886 101.8 55.9375 0.0140129 0.00981882 2024-04-05T02:52:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:10 smithi196 ceph-mon[33169]: pgmap v126: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-04-05T02:52:10.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:10 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1705096844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:10.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:10] "GET /metrics HTTP/1.1" 200 242863 "" "Prometheus/2.43.0" 2024-04-05T02:52:10.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:10 smithi150 ceph-mon[41818]: pgmap v126: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-04-05T02:52:10.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:10 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1705096844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:10.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:10 smithi150 ceph-mon[29558]: pgmap v126: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-04-05T02:52:10.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:10 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1705096844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:10.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 66 16 106868 106852 101.172 60.375 0.00553762 0.00987943 2024-04-05T02:52:11.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 67 16 107828 107812 100.557 60 0.0188409 0.00993961 2024-04-05T02:52:12.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:12 smithi196 ceph-mon[33169]: pgmap v127: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-05T02:52:12.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:12 smithi150 ceph-mon[29558]: pgmap v127: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-05T02:52:12.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:12 smithi150 ceph-mon[41818]: pgmap v127: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-05T02:52:12.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 68 16 108724 108708 99.9021 56 0.0198826 0.0100048 2024-04-05T02:52:12.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:12 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:52:13.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:13 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3484717497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:13.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:13 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3484717497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:13.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:13 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3484717497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:13.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 69 16 109628 109612 99.273 56.5 0.00810515 0.0100684 2024-04-05T02:52:14.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:14 smithi196 ceph-mon[33169]: pgmap v128: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2024-04-05T02:52:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:14 smithi150 ceph-mon[29558]: pgmap v128: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2024-04-05T02:52:14.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:14 smithi150 ceph-mon[41818]: pgmap v128: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 52 MiB/s wr, 829 op/s 2024-04-05T02:52:14.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 70 16 110568 110552 98.6941 58.75 0.012941 0.0101278 2024-04-05T02:52:15.839 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 71 16 111499 111483 98.1235 58.1875 0.0118453 0.0101863 2024-04-05T02:52:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:16 smithi196 ceph-mon[33169]: pgmap v129: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-05T02:52:16.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:16 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/771865986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:16.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:16 smithi150 ceph-mon[29558]: pgmap v129: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-05T02:52:16.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:16 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/771865986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:16.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:16 smithi150 ceph-mon[41818]: pgmap v129: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-04-05T02:52:16.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:16 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/771865986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:16.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 72 16 112385 112369 97.5298 55.375 0.0254257 0.0102478 2024-04-05T02:52:17.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 73 16 113245 113229 96.93 53.75 0.0144815 0.0103121 2024-04-05T02:52:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:18 smithi150 ceph-mon[29558]: pgmap v130: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2024-04-05T02:52:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/822709613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:18 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:18 smithi150 ceph-mon[41818]: pgmap v130: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2024-04-05T02:52:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/822709613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:18 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:18.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:18 smithi196 ceph-mon[33169]: pgmap v130: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s 2024-04-05T02:52:18.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/822709613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:18.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:18 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:18.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 74 16 114139 114123 96.3752 55.875 0.0199708 0.0103717 2024-04-05T02:52:19.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 75 16 115012 114996 95.8176 54.5625 0.0165686 0.0104313 2024-04-05T02:52:20.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:20 smithi150 ceph-mon[29558]: pgmap v131: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2024-04-05T02:52:20.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:20 smithi150 ceph-mon[41818]: pgmap v131: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2024-04-05T02:52:20.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:20] "GET /metrics HTTP/1.1" 200 242891 "" "Prometheus/2.43.0" 2024-04-05T02:52:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:20 smithi196 ceph-mon[33169]: pgmap v131: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2024-04-05T02:52:20.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 76 16 115955 115939 95.3323 58.9375 0.00992966 0.0104841 2024-04-05T02:52:21.458 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3616543974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:21.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:21 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3616543974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:21 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3616543974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:21.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 77 16 116992 116976 94.9359 64.8125 0.00917819 0.0105287 2024-04-05T02:52:22.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:22 smithi150 ceph-mon[29558]: pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-05T02:52:22.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:22 smithi150 ceph-mon[41818]: pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-05T02:52:22.527 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:22 smithi196 ceph-mon[33169]: pgmap v132: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-04-05T02:52:22.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 78 16 117986 117970 94.5152 62.125 0.0339219 0.0105751 2024-04-05T02:52:22.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[35275]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:52:23.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 79 16 118939 118923 94.0727 59.5625 0.010691 0.0106247 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: Upgrade: Updating mgr.x 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: Deploying daemon mgr.x on smithi196 2024-04-05T02:52:24.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2024-04-05T02:52:24.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:23 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3402106270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: Upgrade: Updating mgr.x 2024-04-05T02:52:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:52:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: Deploying daemon mgr.x on smithi196 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3402106270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: Upgrade: Updating mgr.x 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:52:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: Deploying daemon mgr.x on smithi196 2024-04-05T02:52:24.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: pgmap v133: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 48 MiB/s wr, 760 op/s 2024-04-05T02:52:24.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:23 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3402106270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:24.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:52:24.841865+0000 min lat: 0.00110037 max lat: 0.160519 avg lat: 0.0106683 2024-04-05T02:52:24.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:52:24.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 80 16 119957 119941 93.6921 63.625 0.0116213 0.0106683 2024-04-05T02:52:25.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:25 smithi196 systemd[1]: Stopping Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:52:25.820 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:25 smithi196 podman[68553]: 2024-04-05 02:52:25.508494981 +0000 UTC m=+0.181813588 container died 5dcef717d4545091d9f2d599672352420640dc15be48fa1903dcf7b048f0f2b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS) 2024-04-05T02:52:25.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 81 16 120898 120882 93.2614 58.8125 0.00606583 0.0107175 2024-04-05T02:52:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:25 smithi196 ceph-mon[33169]: pgmap v134: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-05T02:52:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:25 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/272835182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:26.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 podman[68553]: 2024-04-05 02:52:26.01145179 +0000 UTC m=+0.684770396 container cleanup 5dcef717d4545091d9f2d599672352420640dc15be48fa1903dcf7b048f0f2b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-05T02:52:26.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 bash[68553]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x 2024-04-05T02:52:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:25 smithi150 ceph-mon[41818]: pgmap v134: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-05T02:52:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:25 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/272835182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:25 smithi150 ceph-mon[29558]: pgmap v134: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-04-05T02:52:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:25 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/272835182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:26.604 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 podman[68569]: 2024-04-05 02:52:26.338769787 +0000 UTC m=+0.827068836 container remove 5dcef717d4545091d9f2d599672352420640dc15be48fa1903dcf7b048f0f2b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-05T02:52:26.604 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-05T02:52:26.605 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-05T02:52:26.605 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 systemd[1]: Stopped Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:52:26.605 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service: Consumed 18.294s CPU time. 2024-04-05T02:52:26.840 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 82 16 121842 121826 92.8435 59 0.0228519 0.0107657 2024-04-05T02:52:27.269 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:26 smithi196 systemd[1]: Starting Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:52:27.618 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:27 smithi196 podman[68685]: 2024-04-05 02:52:27.173343661 +0000 UTC m=+0.040808890 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:52:27.619 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:27 smithi196 podman[68685]: 2024-04-05 02:52:27.298896504 +0000 UTC m=+0.166361734 container create 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-0028525) 2024-04-05T02:52:27.619 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:27 smithi196 podman[68685]: 2024-04-05 02:52:27.618878165 +0000 UTC m=+0.486343397 container init 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-05T02:52:27.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 83 16 122754 122738 92.4116 57 0.015293 0.0108157 2024-04-05T02:52:27.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:27 smithi196 podman[68685]: 2024-04-05 02:52:27.632478801 +0000 UTC m=+0.499944028 container start 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-05T02:52:27.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:27 smithi196 bash[68685]: 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 2024-04-05T02:52:27.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:27 smithi196 systemd[1]: Started Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[41818]: pgmap v135: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[29558]: pgmap v135: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:27 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:27 smithi196 ceph-mon[33169]: pgmap v135: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s 2024-04-05T02:52:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:27 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:27 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:27 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:28.800 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:28.620+0000 7fc8cda21200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:52:28.800 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:28.797+0000 7fc8cda21200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:52:28.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 84 16 123560 123544 91.9112 50.375 0.0169893 0.010875 2024-04-05T02:52:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:28 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4243601922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:29.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:28 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4243601922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:29.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:28 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4243601922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:29.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 85 16 124370 124354 91.4254 50.625 0.0239344 0.010933 2024-04-05T02:52:30.130 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:29 smithi196 ceph-mon[33169]: pgmap v136: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2024-04-05T02:52:30.130 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:29.962+0000 7fc8cda21200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:52:30.131 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:30.129+0000 7fc8cda21200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:52:30.217 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:29 smithi150 ceph-mon[41818]: pgmap v136: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2024-04-05T02:52:30.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:29 smithi150 ceph-mon[29558]: pgmap v136: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 850 op/s 2024-04-05T02:52:30.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:30.371+0000 7fc8cda21200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:52:30.472 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:30] "GET /metrics HTTP/1.1" 200 242891 "" "Prometheus/2.43.0" 2024-04-05T02:52:30.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 86 16 125053 125037 90.8587 42.6875 0.0161818 0.0109997 2024-04-05T02:52:30.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:30.543+0000 7fc8cda21200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:52:30.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:30.694+0000 7fc8cda21200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:52:31.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:30 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2320863090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:31.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:30 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2320863090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:31.366 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:30 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2320863090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:31.367 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:31.156+0000 7fc8cda21200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:52:31.630 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:31.347+0000 7fc8cda21200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:52:31.630 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:31.467+0000 7fc8cda21200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:52:31.630 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:31.629+0000 7fc8cda21200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:52:31.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 87 16 125805 125789 90.3545 47 0.0274439 0.0110624 2024-04-05T02:52:32.217 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:31 smithi196 ceph-mon[33169]: pgmap v137: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:52:32.218 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:31.910+0000 7fc8cda21200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:52:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:31 smithi150 ceph-mon[29558]: pgmap v137: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:52:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:31 smithi150 ceph-mon[41818]: pgmap v137: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-04-05T02:52:32.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 88 16 126618 126602 89.9051 50.8125 0.0162467 0.0111183 2024-04-05T02:52:33.477 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:33 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:33.477 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:33 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:33.477 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:33 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:33.477 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:33 smithi196 ceph-mon[33169]: pgmap v138: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2024-04-05T02:52:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:33.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[29558]: pgmap v138: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2024-04-05T02:52:33.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:33.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:33.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:33.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:33 smithi150 ceph-mon[41818]: pgmap v138: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2024-04-05T02:52:33.825 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:33.593+0000 7fc8cda21200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:52:33.825 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:33.824+0000 7fc8cda21200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:52:33.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 89 16 127452 127436 89.4806 52.125 0.0124434 0.0111703 2024-04-05T02:52:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:34 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1071651324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:34.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:34.389+0000 7fc8cda21200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:52:34.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:34.544+0000 7fc8cda21200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:52:34.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:34 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1071651324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:34.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:34 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1071651324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:34.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 90 7 128277 128270 89.0655 52.125 0.0113095 0.0112227 2024-04-05T02:52:35.062 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:34.869+0000 7fc8cda21200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:52:35.062 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:35.056+0000 7fc8cda21200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:52:35.353 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:35 smithi196 ceph-mon[33169]: pgmap v139: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-05T02:52:35.624 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:35.353+0000 7fc8cda21200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:52:35.624 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:35.623+0000 7fc8cda21200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:52:35.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:35 smithi150 ceph-mon[29558]: pgmap v139: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-05T02:52:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:35 smithi150 ceph-mon[41818]: pgmap v139: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-04-05T02:52:35.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total time run: 90.0129 2024-04-05T02:52:35.841 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total writes made: 128277 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Write size: 65536 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object size: 65536 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Bandwidth (MB/sec): 89.0685 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Bandwidth: 42.9593 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max bandwidth (MB/sec): 250.938 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min bandwidth (MB/sec): 42.6875 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average IOPS: 1425 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev IOPS: 687.35 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max IOPS: 4015 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min IOPS: 683 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average Latency(s): 0.0112232 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Latency(s): 0.00807964 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max latency(s): 0.160519 2024-04-05T02:52:35.842 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min latency(s): 0.00110037 2024-04-05T02:52:35.849 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Cleaning up (deleting benchmark objects) 2024-04-05T02:52:36.314 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:35.884+0000 7fc8cda21200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:52:36.628 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:36 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/864444058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:36.628 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:36 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/864444058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:36 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/864444058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:37 smithi196 ceph-mon[33169]: pgmap v140: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2024-04-05T02:52:37.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:37.232+0000 7fc8cda21200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:52:37.638 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:37 smithi150 ceph-mon[29558]: pgmap v140: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2024-04-05T02:52:37.638 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:37 smithi150 ceph-mon[41818]: pgmap v140: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 44 MiB/s wr, 709 op/s 2024-04-05T02:52:38.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:38.121+0000 7fc8cda21200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:52:38.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:38.258+0000 7fc8cda21200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:52:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-mon[33169]: Standby manager daemon x restarted 2024-04-05T02:52:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-mon[33169]: Standby manager daemon x started 2024-04-05T02:52:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:52:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:52:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:52:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:38.597+0000 7fc8cda21200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:38] ENGINE Bus STARTING 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: CherryPy Checker: 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: The Application mounted at '' has an empty config. 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:38] ENGINE Serving on http://:::9283 2024-04-05T02:52:38.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:38] ENGINE Bus STARTED 2024-04-05T02:52:38.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[41818]: Standby manager daemon x restarted 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[41818]: Standby manager daemon x started 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[29558]: Standby manager daemon x restarted 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[29558]: Standby manager daemon x started 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:52:38.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:52:38.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:52:38.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:38 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:52:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[41818]: mgrmap e21: y(active, since 4m), standbys: x 2024-04-05T02:52:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4226159002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[41818]: pgmap v141: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 45 MiB/s wr, 765 op/s 2024-04-05T02:52:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:39.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[29558]: mgrmap e21: y(active, since 4m), standbys: x 2024-04-05T02:52:39.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4226159002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:39.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:39.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[29558]: pgmap v141: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 45 MiB/s wr, 765 op/s 2024-04-05T02:52:39.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:39 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:40.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:39 smithi196 ceph-mon[33169]: mgrmap e21: y(active, since 4m), standbys: x 2024-04-05T02:52:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:39 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4226159002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:39 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:39 smithi196 ceph-mon[33169]: pgmap v141: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 45 MiB/s wr, 765 op/s 2024-04-05T02:52:40.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:39 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:40.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:40] "GET /metrics HTTP/1.1" 200 242919 "" "Prometheus/2.43.0" 2024-04-05T02:52:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:40 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:41.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:40 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:40 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:40 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:41.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:40 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:41.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:40 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:42.291 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:42 smithi196 ceph-mon[33169]: pgmap v142: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 921 op/s 2024-04-05T02:52:42.291 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:42 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3463945710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:42.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:42 smithi150 ceph-mon[41818]: pgmap v142: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 921 op/s 2024-04-05T02:52:42.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:42 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3463945710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:42.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:42 smithi150 ceph-mon[29558]: pgmap v142: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 44 MiB/s wr, 921 op/s 2024-04-05T02:52:42.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:42 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3463945710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:42.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:52:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:44 smithi196 ceph-mon[33169]: pgmap v143: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 609 op/s 2024-04-05T02:52:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:44 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2831559759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:44.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:44 smithi150 ceph-mon[29558]: pgmap v143: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 609 op/s 2024-04-05T02:52:44.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:44 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2831559759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:44 smithi150 ceph-mon[41818]: pgmap v143: 89 pgs: 89 active+clean; 7.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 609 op/s 2024-04-05T02:52:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:44 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2831559759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:46.340 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:46 smithi196 ceph-mon[33169]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 973 op/s 2024-04-05T02:52:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:46 smithi150 ceph-mon[41818]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 973 op/s 2024-04-05T02:52:46.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:46 smithi150 ceph-mon[29558]: pgmap v144: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 24 MiB/s wr, 973 op/s 2024-04-05T02:52:47.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:47 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1073121205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:47 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1073121205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:47 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1073121205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:48 smithi196 ceph-mon[33169]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 648 op/s 2024-04-05T02:52:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:48 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:48 smithi150 ceph-mon[41818]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 648 op/s 2024-04-05T02:52:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:48 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:48 smithi150 ceph-mon[29558]: pgmap v145: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 648 op/s 2024-04-05T02:52:48.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:48 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:49.556 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:49 smithi150 ceph-mon[41818]: pgmap v146: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 672 op/s 2024-04-05T02:52:49.556 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:49 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/489595334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:49.556 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:49 smithi150 ceph-mon[29558]: pgmap v146: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 672 op/s 2024-04-05T02:52:49.556 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:49 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/489595334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:49.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:49 smithi196 ceph-mon[33169]: pgmap v146: 89 pgs: 89 active+clean; 7.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 4.0 MiB/s wr, 672 op/s 2024-04-05T02:52:49.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:49 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/489595334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:50.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:49.951+0000 7f33f53e3700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-05T02:52:50.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setuser ceph since I am not root 2024-04-05T02:52:50.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ignoring --setgroup ceph since I am not root 2024-04-05T02:52:50.252 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:49] ENGINE Bus STOPPING 2024-04-05T02:52:50.252 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:52:50.252 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:49] ENGINE Bus STOPPED 2024-04-05T02:52:50.252 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:50] ENGINE Bus STARTING 2024-04-05T02:52:50.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.356+0000 7fc833ec6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.356+0000 7fc833ec6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.356+0000 7fc833ec6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.356+0000 7fc833ec6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.356+0000 7fc833ec6700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.361+0000 7fc82b6b5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.361+0000 7fc82b6b5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.361+0000 7fc82b6b5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.361+0000 7fc82b6b5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:52:50.361+0000 7fc82b6b5700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:50] ENGINE Serving on http://:::9283 2024-04-05T02:52:50.638 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:52:50] ENGINE Bus STARTED 2024-04-05T02:52:51.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: osdmap e76: 8 total, 8 up, 8 in 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: mgrmap e22: x(active, starting, since 0.0118832s) 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:52:51.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: Manager daemon x is now available 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:52:51.139 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:52:51.140 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T02:52:51.140 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:50 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T02:52:51.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:50.751+0000 7f8b382051c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:52:51.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:50.893+0000 7f8b382051c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: osdmap e76: 8 total, 8 up, 8 in 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: mgrmap e22: x(active, starting, since 0.0118832s) 2024-04-05T02:52:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:52:51.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: Manager daemon x is now available 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:52:51.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 172.21.15.150:0/3235566730' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: osdmap e76: 8 total, 8 up, 8 in 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.14427 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: mgrmap e22: x(active, starting, since 0.0118832s) 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:52:51.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: Manager daemon x is now available 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:52:51.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:52:51.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:52:51.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T02:52:51.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:50 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-05T02:52:52.154 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:51.890+0000 7f8b382051c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:52:52.154 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:52.038+0000 7f8b382051c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:52:52.154 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: Deploying cephadm binary to smithi150 2024-04-05T02:52:52.154 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: Deploying cephadm binary to smithi196 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: mgrmap e23: x(active, since 1.01785s) 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: pgmap v3: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/134145467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: Deploying cephadm binary to smithi150 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: Deploying cephadm binary to smithi196 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: mgrmap e23: x(active, since 1.01785s) 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: pgmap v3: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:52:52.155 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:51 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/134145467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:52.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: Deploying cephadm binary to smithi150 2024-04-05T02:52:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: Deploying cephadm binary to smithi196 2024-04-05T02:52:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: mgrmap e23: x(active, since 1.01785s) 2024-04-05T02:52:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: pgmap v3: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:52:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:52:52.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:51 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/134145467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:52.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:52:52.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T02:52:52.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:52.153+0000 7f8b382051c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:52:52.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:52.365+0000 7f8b382051c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:52:52.741 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:52.480+0000 7f8b382051c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:52:52.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:52:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:52:52] "GET /metrics HTTP/1.1" 200 33354 "" "Prometheus/2.43.0" 2024-04-05T02:52:53.009 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-mon[29558]: pgmap v4: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:53.009 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:52.741+0000 7f8b382051c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:52:53.010 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:52 smithi150 ceph-mon[41818]: pgmap v4: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:53.262 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:53.009+0000 7f8b382051c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:52:53.262 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:53.133+0000 7f8b382051c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:52:53.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:52 smithi196 ceph-mon[33169]: pgmap v4: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:53.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:53.261+0000 7f8b382051c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:52:53.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:53.386+0000 7f8b382051c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:52:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:53 smithi196 ceph-mon[33169]: mgrmap e24: x(active, since 3s) 2024-04-05T02:52:54.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:53 smithi150 ceph-mon[29558]: mgrmap e24: x(active, since 3s) 2024-04-05T02:52:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:53 smithi150 ceph-mon[41818]: mgrmap e24: x(active, since 3s) 2024-04-05T02:52:55.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:54 smithi150 ceph-mon[29558]: pgmap v5: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:55.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:54 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/592316217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:55.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:54 smithi150 ceph-mon[41818]: pgmap v5: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:55.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:54 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/592316217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:55.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:54.948+0000 7f8b382051c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:52:55.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:55.141+0000 7f8b382051c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:52:55.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:54 smithi196 ceph-mon[33169]: pgmap v5: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:55.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:54 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/592316217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:55.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:55.435+0000 7f8b382051c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:52:56.034 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-mon[41818]: [05/Apr/2024:02:52:55] ENGINE Bus STARTING 2024-04-05T02:52:56.034 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-mon[41818]: [05/Apr/2024:02:52:55] ENGINE Serving on http://172.21.15.196:8765 2024-04-05T02:52:56.035 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:55 smithi150 ceph-mon[29558]: [05/Apr/2024:02:52:55] ENGINE Bus STARTING 2024-04-05T02:52:56.035 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-mon[29558]: [05/Apr/2024:02:52:55] ENGINE Serving on http://172.21.15.196:8765 2024-04-05T02:52:56.035 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:55.754+0000 7f8b382051c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:52:56.318 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:56.034+0000 7f8b382051c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:52:56.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:55 smithi196 ceph-mon[33169]: [05/Apr/2024:02:52:55] ENGINE Bus STARTING 2024-04-05T02:52:56.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:55 smithi196 ceph-mon[33169]: [05/Apr/2024:02:52:55] ENGINE Serving on http://172.21.15.196:8765 2024-04-05T02:52:56.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:56.318+0000 7f8b382051c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:52:57.004 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:56.740+0000 7f8b382051c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:52:57.004 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:56.888+0000 7f8b382051c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:52:57.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:57 smithi196 ceph-mon[33169]: [05/Apr/2024:02:52:55] ENGINE Serving on https://172.21.15.196:7150 2024-04-05T02:52:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:57 smithi196 ceph-mon[33169]: [05/Apr/2024:02:52:55] ENGINE Bus STARTED 2024-04-05T02:52:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:57 smithi196 ceph-mon[33169]: pgmap v6: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:57.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-mon[29558]: [05/Apr/2024:02:52:55] ENGINE Serving on https://172.21.15.196:7150 2024-04-05T02:52:57.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-mon[29558]: [05/Apr/2024:02:52:55] ENGINE Bus STARTED 2024-04-05T02:52:57.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-mon[29558]: pgmap v6: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:57.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-mon[41818]: [05/Apr/2024:02:52:55] ENGINE Serving on https://172.21.15.196:7150 2024-04-05T02:52:57.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-mon[41818]: [05/Apr/2024:02:52:55] ENGINE Bus STARTED 2024-04-05T02:52:57.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-mon[41818]: pgmap v6: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail 2024-04-05T02:52:57.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:57 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:57.505+0000 7f8b382051c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:52:58.287 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:58 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/967599100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:58 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/967599100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:58 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/967599100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:52:59.017 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:58 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:58.745+0000 7f8b382051c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:52:59.300 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:52:59 smithi196 ceph-mon[33169]: pgmap v7: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 559 op/s 2024-04-05T02:52:59.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:52:59 smithi150 ceph-mon[41818]: pgmap v7: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 559 op/s 2024-04-05T02:52:59.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:52:59 smithi150 ceph-mon[29558]: pgmap v7: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 559 op/s 2024-04-05T02:52:59.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:59 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:59.201+0000 7f8b382051c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:52:59.960 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:59 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:59.649+0000 7f8b382051c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:53:00.217 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[29558]: Standby manager daemon y started 2024-04-05T02:53:00.218 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T02:53:00.218 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:53:00.218 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T02:53:00.218 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:53:00.218 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2668474138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[41818]: Standby manager daemon y started 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2668474138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:00.219 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:59 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:52:59.960+0000 7f8b382051c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:52:59 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:52:59] ENGINE Bus STARTING 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: CherryPy Checker: 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: The Application mounted at '' has an empty config. 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:53:00] ENGINE Serving on http://:::9283 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: [05/Apr/2024:02:53:00] ENGINE Bus STARTED 2024-04-05T02:53:00.220 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:53:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:00 smithi196 ceph-mon[33169]: Standby manager daemon y started 2024-04-05T02:53:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:00 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T02:53:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:00 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:53:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:00 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T02:53:00.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:00 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/727277224' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:53:00.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:00 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2668474138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:01.227 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:53:00.819Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=7 2024-04-05T02:53:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:01 smithi150 ceph-mon[29558]: pgmap v8: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 434 op/s 2024-04-05T02:53:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:01 smithi150 ceph-mon[29558]: mgrmap e25: x(active, since 10s), standbys: y 2024-04-05T02:53:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:01 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:53:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:01 smithi150 ceph-mon[41818]: pgmap v8: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 434 op/s 2024-04-05T02:53:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:01 smithi150 ceph-mon[41818]: mgrmap e25: x(active, since 10s), standbys: y 2024-04-05T02:53:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:01 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:53:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:01 smithi196 ceph-mon[33169]: pgmap v8: 89 pgs: 89 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 434 op/s 2024-04-05T02:53:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:01 smithi196 ceph-mon[33169]: mgrmap e25: x(active, since 10s), standbys: y 2024-04-05T02:53:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:01 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:53:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:02] "GET /metrics HTTP/1.1" 200 33354 "" "Prometheus/2.43.0" 2024-04-05T02:53:03.325 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:03 smithi150 ceph-mon[41818]: pgmap v9: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 714 op/s 2024-04-05T02:53:03.325 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:03 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2103564718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:03.326 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:03 smithi150 ceph-mon[29558]: pgmap v9: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 714 op/s 2024-04-05T02:53:03.326 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:03 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2103564718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:03.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:03 smithi196 ceph-mon[33169]: pgmap v9: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 714 op/s 2024-04-05T02:53:03.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:03 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2103564718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[29558]: pgmap v10: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-05T02:53:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2342217827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[41818]: pgmap v10: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:05 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2342217827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:06.002 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:05 smithi196 ceph-mon[33169]: pgmap v10: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-05T02:53:06.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:05 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:06.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:05 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:06.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:05 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:06.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:05 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2342217827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: pgmap v11: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi150 to 2480M 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: pgmap v11: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi150 to 2480M 2024-04-05T02:53:07.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: pgmap v11: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 653 op/s 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi150 to 2480M 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:07.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:08 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1599821692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:08 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:08 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:08.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:08 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1599821692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:08.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:08 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:08.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:08 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:08.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:08 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1599821692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:08.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:08 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:08.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:08 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:09.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:09 smithi196 ceph-mon[33169]: pgmap v12: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 1.00k op/s 2024-04-05T02:53:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:09 smithi150 ceph-mon[29558]: pgmap v12: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 1.00k op/s 2024-04-05T02:53:09.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:09 smithi150 ceph-mon[41818]: pgmap v12: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 1.00k op/s 2024-04-05T02:53:10.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:53:11.499 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:11 smithi196 ceph-mon[33169]: pgmap v13: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:11.499 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:11 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/571209387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:11.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:11 smithi150 ceph-mon[29558]: pgmap v13: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:11.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:11 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/571209387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:11.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:11 smithi150 ceph-mon[41818]: pgmap v13: 89 pgs: 89 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:11 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/571209387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:12.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:12 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:12] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.43.0" 2024-04-05T02:53:13.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:13 smithi196 ceph-mon[33169]: pgmap v14: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-05T02:53:13.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:13 smithi150 ceph-mon[41818]: pgmap v14: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-05T02:53:13.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:13 smithi150 ceph-mon[29558]: pgmap v14: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2808266030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2808266030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:53:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:14 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2808266030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-05T02:53:14.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:14.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:53:14.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:14 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:53:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: Adjusting osd_memory_target on smithi196 to 2480M 2024-04-05T02:53:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: pgmap v15: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: Adjusting osd_memory_target on smithi196 to 2480M 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: pgmap v15: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:53:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:15 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: Adjusting osd_memory_target on smithi196 to 2480M 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: pgmap v15: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:15 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:53:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:53:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:53:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:16 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1945066799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.896 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1945066799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.897 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.898 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.898 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.898 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:16.898 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:16 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1945066799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:17 smithi150 ceph-mon[29558]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T02:53:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:17 smithi150 ceph-mon[29558]: Reconfiguring daemon alertmanager.a on smithi150 2024-04-05T02:53:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:17 smithi150 ceph-mon[29558]: pgmap v16: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:53:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:17 smithi150 ceph-mon[41818]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T02:53:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:17 smithi150 ceph-mon[41818]: Reconfiguring daemon alertmanager.a on smithi150 2024-04-05T02:53:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:17 smithi150 ceph-mon[41818]: pgmap v16: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:53:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:17 smithi196 ceph-mon[33169]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-05T02:53:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:17 smithi196 ceph-mon[33169]: Reconfiguring daemon alertmanager.a on smithi150 2024-04-05T02:53:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:17 smithi196 ceph-mon[33169]: pgmap v16: 89 pgs: 89 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:53:19.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:19 smithi150 ceph-mon[29558]: pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:53:19.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:19 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3037700536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:19 smithi150 ceph-mon[41818]: pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:53:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:19 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3037700536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:19.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:19 smithi196 ceph-mon[33169]: pgmap v17: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:53:19.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:19 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3037700536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:20.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:53:20.478 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:20 smithi150 systemd[1]: Stopping Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:20.832 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:20 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:20.832 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:20 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:20.833 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[78510]: ts=2024-04-05T02:53:20.791Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-05T02:53:20.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:20 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:21.227 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:20 smithi150 podman[91474]: 2024-04-05 02:53:20.832748068 +0000 UTC m=+0.198632510 container died 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:21.583 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 podman[91474]: 2024-04-05 02:53:21.359560035 +0000 UTC m=+0.725444469 container cleanup 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:21.583 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 bash[91474]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-05T02:53:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:21 smithi196 ceph-mon[33169]: pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-05T02:53:21.894 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:21 smithi150 ceph-mon[41818]: pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-05T02:53:21.894 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:21 smithi150 ceph-mon[29558]: pgmap v18: 89 pgs: 89 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-05T02:53:21.894 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 podman[91493]: 2024-04-05 02:53:21.636471552 +0000 UTC m=+0.798049170 container remove 05286344493f23ae76649652d3faa0125b403243a3f021348dfa6b661c43abec (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:21.894 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 podman[91493]: 2024-04-05 02:53:21.721110804 +0000 UTC m=+0.882688429 volume remove 8ad84c32e514402ebb9a0c2a5ad2e705b3702f6a337865747ef4c42f1ad6f685 2024-04-05T02:53:22.227 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-05T02:53:22.227 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 systemd[1]: Stopped Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:22.227 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@alertmanager.a.service: Consumed 1.423s CPU time. 2024-04-05T02:53:22.228 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:21 smithi150 systemd[1]: Starting Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:22.591 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 podman[91602]: 2024-04-05 02:53:22.151908174 +0000 UTC m=+0.039620407 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-05T02:53:22.591 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 podman[91602]: 2024-04-05 02:53:22.27225292 +0000 UTC m=+0.159965147 volume create ef885945dca93dbaddb4508256098671422b25ca504f798def71d2d21e72fc7a 2024-04-05T02:53:22.591 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 podman[91602]: 2024-04-05 02:53:22.406641332 +0000 UTC m=+0.294353558 container create 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:22.591 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:22] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.43.0" 2024-04-05T02:53:22.592 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:22.317Z caller=notifier.go:532 level=error component=notifier alertmanager=http://smithi150.front.sepia.ceph.com:9093/api/v2/alerts count=1 msg="Error sending alert" err="Post \"http://smithi150.front.sepia.ceph.com:9093/api/v2/alerts\": dial tcp 172.21.15.150:9093: connect: connection refused" 2024-04-05T02:53:22.849 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1792769412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:22.849 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1792769412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:22.849 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 podman[91602]: 2024-04-05 02:53:22.769093258 +0000 UTC m=+0.656805493 container init 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:22.850 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 podman[91602]: 2024-04-05 02:53:22.779862368 +0000 UTC m=+0.667574590 container start 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:22.850 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.800Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-05T02:53:22.850 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.800Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-05T02:53:22.850 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.801Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.150 port=9094 2024-04-05T02:53:22.850 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.803Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-05T02:53:22.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:22 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1792769412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:23.228 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.849Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T02:53:23.228 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.851Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-05T02:53:23.229 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.855Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-05T02:53:23.229 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:22.856Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-05T02:53:23.229 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 bash[91602]: 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d 2024-04-05T02:53:23.229 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:22 smithi150 systemd[1]: Started Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:23.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:23 smithi196 ceph-mon[33169]: pgmap v19: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:23.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:23 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:23.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:23 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:23.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:23 smithi150 ceph-mon[41818]: pgmap v19: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:23.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:23 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:23.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:23 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:23.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:23 smithi150 ceph-mon[29558]: pgmap v19: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:23.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:23 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:23.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:23 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:23.978 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:23 smithi150 systemd[1]: Stopping Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:24.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:24 smithi196 ceph-mon[33169]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-05T02:53:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:24 smithi196 ceph-mon[33169]: Reconfiguring daemon node-exporter.a on smithi150 2024-04-05T02:53:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:24 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3638434502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-mon[29558]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-05T02:53:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-mon[29558]: Reconfiguring daemon node-exporter.a on smithi150 2024-04-05T02:53:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3638434502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-mon[41818]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-05T02:53:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-mon[41818]: Reconfiguring daemon node-exporter.a on smithi150 2024-04-05T02:53:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3638434502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:24.978 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:24 smithi150 podman[91780]: 2024-04-05 02:53:24.723781129 +0000 UTC m=+0.637914365 container died ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:24.978 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:24.804Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000900234s 2024-04-05T02:53:25.417 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 podman[91780]: 2024-04-05 02:53:25.140834371 +0000 UTC m=+1.054967602 container cleanup ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:25.417 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 bash[91780]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a 2024-04-05T02:53:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:25 smithi150 ceph-mon[29558]: pgmap v20: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-05T02:53:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:25 smithi150 ceph-mon[41818]: pgmap v20: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-05T02:53:25.728 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 podman[91797]: 2024-04-05 02:53:25.417645198 +0000 UTC m=+0.690956098 container remove ebab866c2b05972462bb9fc9aeb2cf4a0122a69371f5f7d55d5504e6a5e30e18 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:25.728 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-05T02:53:25.728 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-05T02:53:25.728 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 systemd[1]: Stopped Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:25.728 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service: Consumed 2.599s CPU time. 2024-04-05T02:53:25.728 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 systemd[1]: Starting Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:25.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:25 smithi196 ceph-mon[33169]: pgmap v20: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-05T02:53:26.227 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 podman[91881]: 2024-04-05 02:53:25.826094939 +0000 UTC m=+0.028641624 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T02:53:26.228 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:25 smithi150 podman[91881]: 2024-04-05 02:53:25.935694282 +0000 UTC m=+0.138240939 container create dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:26.666 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 podman[91881]: 2024-04-05 02:53:26.388710867 +0000 UTC m=+0.591257534 container init dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:26.666 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 podman[91881]: 2024-04-05 02:53:26.40286141 +0000 UTC m=+0.605408064 container start dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:26.666 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.406Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T02:53:26.666 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.406Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.407Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.407Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.408Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.408Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.408Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T02:53:26.667 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T02:53:26.668 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.409Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T02:53:26.669 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T02:53:26.670 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T02:53:26.670 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T02:53:26.670 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T02:53:26.670 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T02:53:26.670 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a[91892]: ts=2024-04-05T02:53:26.410Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T02:53:26.670 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 bash[91881]: dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 2024-04-05T02:53:26.977 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 02:53:26 smithi150 systemd[1]: Started Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[41818]: pgmap v21: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-05T02:53:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3369697303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:27.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[29558]: pgmap v21: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-05T02:53:27.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:27.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:27.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:27 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3369697303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:28.051 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:27 smithi196 ceph-mon[33169]: pgmap v21: 89 pgs: 89 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2024-04-05T02:53:28.051 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:27 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:28.051 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:27 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:28.051 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:27 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3369697303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:28 smithi150 ceph-mon[29558]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:53:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:28 smithi150 ceph-mon[29558]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:53:28.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:28 smithi150 ceph-mon[41818]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:53:28.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:28 smithi150 ceph-mon[41818]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:53:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:28 smithi196 ceph-mon[33169]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:53:29.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:28 smithi196 ceph-mon[33169]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:53:30.039 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:29 smithi196 ceph-mon[33169]: pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:53:30.213 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:29 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/548411863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:30.216 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:29 smithi150 ceph-mon[29558]: pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:53:30.216 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:29 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/548411863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:30.216 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:29 smithi150 ceph-mon[41818]: pgmap v22: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:53:30.216 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/548411863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:30.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[29812]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:53:30.937 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:30 smithi196 ceph-mon[33169]: pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-05T02:53:31.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:30 smithi150 ceph-mon[29558]: pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-05T02:53:31.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:30 smithi150 ceph-mon[41818]: pgmap v23: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-04-05T02:53:32.386 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 systemd[1]: Stopping Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:32.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T02:53:32.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T02:53:32.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T02:53:32.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.664Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.668Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.668Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[65685]: ts=2024-04-05T02:53:32.668Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T02:53:32.888 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:32 smithi196 podman[76702]: 2024-04-05 02:53:32.675548586 +0000 UTC m=+0.162362622 container died 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:32.889 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:32] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.43.0" 2024-04-05T02:53:33.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:33 smithi150 ceph-mon[41818]: pgmap v24: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:53:33.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:33 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4153199600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:33.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:33 smithi150 ceph-mon[29558]: pgmap v24: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:53:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:33 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4153199600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:33.228 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:53:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:53:32.806Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003039263s 2024-04-05T02:53:33.386 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 podman[76702]: 2024-04-05 02:53:33.058249615 +0000 UTC m=+0.545063644 container cleanup 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:33.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 bash[76702]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T02:53:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:33 smithi196 ceph-mon[33169]: pgmap v24: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:53:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:33 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4153199600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:33.886 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 podman[76719]: 2024-04-05 02:53:33.393468202 +0000 UTC m=+0.706135906 container remove 036e6d335f1a38e13eb0738c22efee07442dd6d9244fed7761622dce39914ed0 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:33.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-05T02:53:33.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 systemd[1]: Stopped Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:33.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service: Consumed 2.128s CPU time. 2024-04-05T02:53:33.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 systemd[1]: Starting Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:34.274 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 podman[76803]: 2024-04-05 02:53:33.864338844 +0000 UTC m=+0.039025488 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T02:53:34.274 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:33 smithi196 podman[76803]: 2024-04-05 02:53:33.995193602 +0000 UTC m=+0.169880226 container create 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:34.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 podman[76803]: 2024-04-05 02:53:34.274362198 +0000 UTC m=+0.449048817 container init 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:34.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 podman[76803]: 2024-04-05 02:53:34.283636317 +0000 UTC m=+0.458322942 container start 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:53:34.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.332Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.333Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.333Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi196 (none))" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.333Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.333Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.337Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.338Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.339Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.344Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.348Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.348Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.61µs 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.348Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.349Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-05T02:53:34.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 bash[76803]: 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 systemd[1]: Started Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.405Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.407Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.407Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.423µs wal_replay_duration=58.875822ms wbl_replay_duration=147ns total_replay_duration=58.93077ms 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.412Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.412Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.412Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.448Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=36.232095ms db_storage=1.26µs remote_storage=2.048µs web_handler=450ns query_engine=852ns scrape=678.371µs scrape_sd=129.911µs notify=14.838µs notify_sd=15.268µs rules=35.020884ms tracing=7.438µs 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.448Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T02:53:34.639 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:34.449Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T02:53:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: pgmap v25: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-05T02:53:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: Reconfiguring daemon grafana.a on smithi196 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1158031716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: pgmap v25: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: Reconfiguring daemon grafana.a on smithi196 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1158031716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:35 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: pgmap v25: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: Reconfiguring daemon grafana.a on smithi196 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1158031716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:35 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:37.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:37 smithi150 ceph-mon[41818]: pgmap v26: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-05T02:53:37.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:37 smithi150 ceph-mon[29558]: pgmap v26: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-05T02:53:37.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:37 smithi196 ceph-mon[33169]: pgmap v26: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 705 op/s 2024-04-05T02:53:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:38 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2641971454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:38.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:38 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2641971454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:38.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:38 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2641971454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:39 smithi150 ceph-mon[41818]: pgmap v27: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-05T02:53:39.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:39 smithi150 ceph-mon[29558]: pgmap v27: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-05T02:53:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:39 smithi196 ceph-mon[33169]: pgmap v27: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-05T02:53:40.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:40 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2956129387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:40.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:40 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2956129387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:40.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:40 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2956129387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:40.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:40] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.43.0" 2024-04-05T02:53:41.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:41 smithi150 ceph-mon[41818]: pgmap v28: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-05T02:53:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:41 smithi150 ceph-mon[29558]: pgmap v28: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-05T02:53:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:41 smithi196 ceph-mon[33169]: pgmap v28: 89 pgs: 89 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 696 op/s 2024-04-05T02:53:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:43 smithi196 ceph-mon[33169]: pgmap v29: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:43 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/813127236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:43.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:43 smithi150 ceph-mon[29558]: pgmap v29: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:43.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:43 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/813127236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:43 smithi150 ceph-mon[41818]: pgmap v29: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:43 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/813127236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:45 smithi196 ceph-mon[33169]: pgmap v30: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-04-05T02:53:45.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:45 smithi150 ceph-mon[41818]: pgmap v30: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-04-05T02:53:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:45 smithi150 ceph-mon[29558]: pgmap v30: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-04-05T02:53:46.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:46 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3552533334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:46 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3552533334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:46.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:46 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3552533334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:47.496 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:53:47.205+0000 7fc853705700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:47.496 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:47.496 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:47.496 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Getting image source signatures 2024-04-05T02:53:47.496 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: Traceback (most recent call last): 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-05T02:53:47.497 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: return self.event_loop.get_result(coro, timeout) 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: return future.result(timeout) 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: return self.__get_result() 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: raise self._exception 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: config_blobs=daemon_spec.final_config, 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-05T02:53:47.498 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Getting image source signatures 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:47.499 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:47 smithi196 ceph-mon[33169]: pgmap v31: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-04-05T02:53:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:47 smithi150 ceph-mon[29558]: pgmap v31: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-04-05T02:53:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:47 smithi150 ceph-mon[41818]: pgmap v31: 89 pgs: 89 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2024-04-05T02:53:48.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:48 smithi196 systemd[1]: Stopping Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:48 smithi196 ceph-mon[33169]: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Getting image source signatures 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:48.777 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: Traceback (most recent call last): 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: return future.result(timeout) 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-05T02:53:48.778 INFO:journalctl@ceph.mon.b.smithi196.stdout: return self.__get_result() 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: raise self._exception 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: config_blobs=daemon_spec.final_config, 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Getting image source signatures 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:48.779 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:48 smithi196 ceph-mon[33169]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:48 smithi196 ceph-mon[33169]: Reconfiguring daemon node-exporter.b on smithi196 2024-04-05T02:53:48.780 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:48 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3845456296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:48.780 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:53:48.781 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:48 smithi196 podman[77090]: 2024-04-05 02:53:48.457902781 +0000 UTC m=+0.169147693 container died 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:53:48.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[41818]: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:48.977 INFO:journalctl@ceph.mon.c.smithi150.stdout: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Getting image source signatures 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout: Traceback (most recent call last): 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: return future.result(timeout) 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: return self.__get_result() 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: raise self._exception 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: config_blobs=daemon_spec.final_config, 2024-04-05T02:53:48.979 INFO:journalctl@ceph.mon.c.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[29558]: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Getting image source signatures 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:48.980 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: Traceback (most recent call last): 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1114, in _check_daemons 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2390, in _daemon_action 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 747, in wait_async 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: return future.result(timeout) 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-05T02:53:48.981 INFO:journalctl@ceph.mon.a.smithi150.stdout: return self.__get_result() 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.a.smithi150.stdout: raise self._exception 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1374, in _create_daemon 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.a.smithi150.stdout: config_blobs=daemon_spec.final_config, 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.a.smithi150.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1685, in _run_cephadm 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Getting image source signatures 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:48.982 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[41818]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[41818]: Reconfiguring daemon node-exporter.b on smithi196 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3845456296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:48.983 INFO:journalctl@ceph.mon.a.smithi150.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon grafana.a ... 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: Non-zero exit code 125 from /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Trying to pull quay.io/ceph/grafana:9.4.12... 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Getting image source signatures 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:2fe2fda407f1e431cc4fb109fda5fe35c1db7679d5aaad9a525d0426444c7ba2 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:f56be85fc22e46face30e2c3de3f7fe7c15f8fd7c4e5add29d7f64b87abdaa09 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:4ecb983198a1742dd468a170119523c070034ac278cb52ecbbe647d89baeceed 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:da42aa123f0896b6f42a4aea54a1a4c1c8ea17f2552efa1fbdeb6b58cae77cf5 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:3d131109d2cc40ca95c36a5a68109ab7cc1f8bf681b896c2a2efe9ebd9016ed9 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:8827bb7f20fe474546a182a813fd48d1927630c27c09d1a942560918dfc4aee8 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:eb67b3ceb6a9b5c0efb69d314abb56bc1a5f24e0e33702c163dc5b1632739780 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Copying blob sha256:fdb0e4e77fcb4d0e3e45942b79a214201c0f9d2c51da5dae6ab3bd3dd4fa5245 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: stat: stderr Error: reading blob sha256:afae8f6af3d8c3b9a1dc3022e7170da6dcdab9a6174bccc9d61f90ef96398fb0: fetching blob: received unexpected HTTP status: 502 Bad Gateway 2024-04-05T02:53:48.984 INFO:journalctl@ceph.mon.a.smithi150.stdout: ERROR: Failed to extract uid/gid for path /var/lib/grafana: Failed command: /usr/bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint stat --init -e CONTAINER_IMAGE=quay.io/ceph/grafana:9.4.12 -e NODE_NAME=smithi196 quay.io/ceph/grafana:9.4.12 -c %u %g /var/lib/grafana 2024-04-05T02:53:48.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[29558]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-05T02:53:48.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[29558]: Reconfiguring daemon node-exporter.b on smithi196 2024-04-05T02:53:48.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:48 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3845456296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:49.136 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:48 smithi196 podman[77090]: 2024-04-05 02:53:48.868208044 +0000 UTC m=+0.579452955 container cleanup 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:53:49.137 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:48 smithi196 bash[77090]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b 2024-04-05T02:53:49.510 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 podman[77104]: 2024-04-05 02:53:49.161785028 +0000 UTC m=+0.705915630 container remove 1d7f72da9f2d4c0510729b1c852a2a46a1aa37475139d282fcf1e4409429373a (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:53:49.510 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-05T02:53:49.510 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-05T02:53:49.511 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 systemd[1]: Stopped Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:49.511 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service: Consumed 2.591s CPU time. 2024-04-05T02:53:49.511 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 systemd[1]: Starting Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:49.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:49 smithi150 ceph-mon[29558]: pgmap v32: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:49.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:49 smithi150 ceph-mon[41818]: pgmap v32: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:49.795 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 podman[77184]: 2024-04-05 02:53:49.615581401 +0000 UTC m=+0.039009936 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-05T02:53:49.795 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:49 smithi196 ceph-mon[33169]: pgmap v32: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:53:50.081 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:49 smithi196 podman[77184]: 2024-04-05 02:53:49.795980501 +0000 UTC m=+0.219409020 container create 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 podman[77184]: 2024-04-05 02:53:50.081540113 +0000 UTC m=+0.504968648 container init 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 podman[77184]: 2024-04-05 02:53:50.095190461 +0000 UTC m=+0.518618988 container start 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.099Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.099Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.100Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.100Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.100Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-05T02:53:50.387 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.100Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=arp 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-05T02:53:50.388 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=edac 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-05T02:53:50.389 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=os 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=stat 2024-04-05T02:53:50.390 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=time 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=uname 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.101Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.102Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.102Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b[77196]: ts=2024-04-05T02:53:50.102Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 bash[77184]: 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 2024-04-05T02:53:50.391 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 02:53:50 smithi196 systemd[1]: Started Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:53:50.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:53:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:53:50] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.43.0" 2024-04-05T02:53:50.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:50.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:50.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:50.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:53:50.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:53:51.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[29558]: pgmap v33: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-04-05T02:53:51.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:53:51.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:53:51.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[29558]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T02:53:51.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[29558]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T02:53:51.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1445872904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:51.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[41818]: pgmap v33: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-04-05T02:53:51.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:53:51.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:53:51.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[41818]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T02:53:51.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[41818]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T02:53:51.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:51 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1445872904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:51.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:51 smithi196 ceph-mon[33169]: pgmap v33: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 707 op/s 2024-04-05T02:53:51.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:51 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-05T02:53:51.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:51 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:53:51.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:51 smithi196 ceph-mon[33169]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T02:53:51.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:51 smithi196 ceph-mon[33169]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-05T02:53:51.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:51 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1445872904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:53.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:53 smithi196 ceph-mon[33169]: pgmap v34: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:53:53.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:53 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:53.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:53 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:53:53.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:53 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:53:53.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:53 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:53:53.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:53 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:53:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[29558]: pgmap v34: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[41818]: pgmap v34: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:53.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:53:53.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:53:53.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:53:53.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:53 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:53:54.796 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:54 smithi150 ceph-mon[41818]: Upgrade: Updating mgr.y 2024-04-05T02:53:54.796 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:54 smithi150 ceph-mon[41818]: Deploying daemon mgr.y on smithi150 2024-04-05T02:53:54.796 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:54 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/921775367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:54.796 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:54 smithi150 ceph-mon[29558]: Upgrade: Updating mgr.y 2024-04-05T02:53:54.796 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:54 smithi150 ceph-mon[29558]: Deploying daemon mgr.y on smithi150 2024-04-05T02:53:54.796 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:54 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/921775367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:54 smithi196 ceph-mon[33169]: Upgrade: Updating mgr.y 2024-04-05T02:53:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:54 smithi196 ceph-mon[33169]: Deploying daemon mgr.y on smithi150 2024-04-05T02:53:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:54 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/921775367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:55.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:55 smithi196 ceph-mon[33169]: pgmap v35: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:55.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:55 smithi150 ceph-mon[41818]: pgmap v35: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:55.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:55 smithi150 ceph-mon[29558]: pgmap v35: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:56.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:56 smithi150 systemd[1]: Stopping Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:56.867 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:56 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1100391636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:56.868 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:56 smithi150 podman[92562]: 2024-04-05 02:53:56.549146345 +0000 UTC m=+0.188372307 container died be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-05T02:53:56.868 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:56 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1100391636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:56 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1100391636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:57.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 podman[92562]: 2024-04-05 02:53:57.067033869 +0000 UTC m=+0.706259823 container cleanup be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, ceph=True, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-05T02:53:57.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 bash[92562]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y 2024-04-05T02:53:57.814 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:57 smithi150 ceph-mon[41818]: pgmap v36: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:57.814 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:57 smithi150 ceph-mon[29558]: pgmap v36: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:57.814 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 podman[92582]: 2024-04-05 02:53:57.561128821 +0000 UTC m=+1.014886271 container remove be1d8eb95a040702611c3ec6e707ab45f9758e042470509c6e09c72635d76a71 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403) 2024-04-05T02:53:57.814 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-05T02:53:57.815 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service: Failed with result 'exit-code'. 2024-04-05T02:53:57.815 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 systemd[1]: Stopped Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:57.815 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:57 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service: Consumed 1min 1.995s CPU time. 2024-04-05T02:53:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:57 smithi196 ceph-mon[33169]: pgmap v36: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 677 op/s 2024-04-05T02:53:58.413 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 systemd[1]: Starting Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:53:58.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 podman[92702]: 2024-04-05 02:53:58.316244999 +0000 UTC m=+0.033080916 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:53:58.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 podman[92702]: 2024-04-05 02:53:58.445737919 +0000 UTC m=+0.162573867 container create 0d5b82cc2eb247424834a171d45a00eb9d8eb40ef3c40b80809577b1788e658e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True) 2024-04-05T02:53:59.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:53:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:53:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:53:59.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 podman[92702]: 2024-04-05 02:53:58.741288977 +0000 UTC m=+0.458124885 container init 0d5b82cc2eb247424834a171d45a00eb9d8eb40ef3c40b80809577b1788e658e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-05T02:53:59.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 podman[92702]: 2024-04-05 02:53:58.750631679 +0000 UTC m=+0.467467583 container start 0d5b82cc2eb247424834a171d45a00eb9d8eb40ef3c40b80809577b1788e658e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403) 2024-04-05T02:53:59.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 bash[92702]: 0d5b82cc2eb247424834a171d45a00eb9d8eb40ef3c40b80809577b1788e658e 2024-04-05T02:53:59.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:58 smithi150 systemd[1]: Started Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:53:59.854 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:53:59.685+0000 7f84b8d9b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:53:59.854 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:53:59.851+0000 7f84b8d9b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:53:59.854 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[41818]: pgmap v37: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-05T02:53:59.854 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:59.854 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:59.854 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:53:59.855 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1104122737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:59.855 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[29558]: pgmap v37: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-05T02:53:59.855 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:59.855 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:59.855 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:53:59.855 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:53:59 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1104122737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:53:59.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:59 smithi196 ceph-mon[33169]: pgmap v37: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-04-05T02:53:59.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:59 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:59.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:59 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:53:59.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:59 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:53:59.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:53:59 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1104122737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:00.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:00 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:54:00] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.43.0" 2024-04-05T02:54:01.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:00 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:01.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:00 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:01.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:00 smithi196 ceph-mon[33169]: pgmap v38: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-05T02:54:01.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:00.931+0000 7f84b8d9b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:54:01.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:01.074+0000 7f84b8d9b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:54:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-mon[41818]: pgmap v38: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-05T02:54:01.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:01.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:01.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:00 smithi150 ceph-mon[29558]: pgmap v38: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s 2024-04-05T02:54:01.606 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:01.314+0000 7f84b8d9b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:54:01.606 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:01.467+0000 7f84b8d9b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:54:01.606 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:01.604+0000 7f84b8d9b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:54:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:02 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4029398875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:02.402 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:02.073+0000 7f84b8d9b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:54:02.403 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:02.249+0000 7f84b8d9b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:54:02.403 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4029398875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:02.403 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4029398875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:02.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:02.402+0000 7f84b8d9b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:54:02.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:02.562+0000 7f84b8d9b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:54:03.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:03 smithi150 ceph-mon[41818]: pgmap v39: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:54:03.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:03 smithi150 ceph-mon[29558]: pgmap v39: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:54:03.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:02.833+0000 7f84b8d9b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:54:03.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:03 smithi196 ceph-mon[33169]: pgmap v39: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:54:03.977 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:54:03 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:54:03.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T02:54:04.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:04.470+0000 7f84b8d9b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:54:04.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:04.692+0000 7f84b8d9b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:54:05.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-mon[41818]: pgmap v40: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:54:05.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2528345592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:05.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-mon[29558]: pgmap v40: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:54:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2528345592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:05 smithi196 ceph-mon[33169]: pgmap v40: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:54:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:05 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2528345592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:05.698 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:05.232+0000 7f84b8d9b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:54:05.698 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:05.395+0000 7f84b8d9b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:54:05.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:05.696+0000 7f84b8d9b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:54:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:06 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:06 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:06 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:06 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:06 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:06.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:06.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:05 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:05.993+0000 7f84b8d9b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:54:06.898 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:06.488+0000 7f84b8d9b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:54:07.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:06 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:06.895+0000 7f84b8d9b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:54:07.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-mon[41818]: pgmap v41: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:54:07.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:07.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-mon[29558]: pgmap v41: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:54:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:07.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:07.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:07.353+0000 7f84b8d9b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:54:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:07 smithi196 ceph-mon[33169]: pgmap v41: 89 pgs: 89 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:54:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:07 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:08.676 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:08 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2584200438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:08.677 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:08 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2584200438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:08.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:08 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2584200438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:08.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:54:08 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:54:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:54:08.930 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:08 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:08.805+0000 7f84b8d9b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:54:09.686 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:09 smithi150 ceph-mon[41818]: pgmap v42: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-05T02:54:09.687 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:09 smithi150 ceph-mon[29558]: pgmap v42: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-05T02:54:09.858 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:09 smithi196 ceph-mon[33169]: pgmap v42: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 982 op/s 2024-04-05T02:54:09.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:09 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:09.814+0000 7f84b8d9b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:54:10.324 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:09 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:09.998+0000 7f84b8d9b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:54:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2840432482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[41818]: Standby manager daemon y restarted 2024-04-05T02:54:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[41818]: Standby manager daemon y started 2024-04-05T02:54:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T02:54:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2840432482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[29558]: Standby manager daemon y restarted 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[29558]: Standby manager daemon y started 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:10.401+0000 7f84b8d9b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:10] ENGINE Bus STARTING 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: CherryPy Checker: 2024-04-05T02:54:10.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: The Application mounted at '' has an empty config. 2024-04-05T02:54:10.729 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:10.729 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:10] ENGINE Serving on http://:::9283 2024-04-05T02:54:10.729 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:10] ENGINE Bus STARTED 2024-04-05T02:54:10.765 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:10 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:54:10] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.43.0" 2024-04-05T02:54:10.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:10 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2840432482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:10.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:10 smithi196 ceph-mon[33169]: Standby manager daemon y restarted 2024-04-05T02:54:10.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:10 smithi196 ceph-mon[33169]: Standby manager daemon y started 2024-04-05T02:54:10.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:10 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-05T02:54:10.766 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:10 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:54:11.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[29558]: pgmap v43: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2024-04-05T02:54:11.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T02:54:11.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[29558]: mgrmap e26: x(active, since 80s), standbys: y 2024-04-05T02:54:11.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:54:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[41818]: pgmap v43: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2024-04-05T02:54:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T02:54:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[41818]: mgrmap e26: x(active, since 80s), standbys: y 2024-04-05T02:54:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:11 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:54:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:11 smithi196 ceph-mon[33169]: pgmap v43: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2024-04-05T02:54:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:11 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-05T02:54:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:11 smithi196 ceph-mon[33169]: mgrmap e26: x(active, since 80s), standbys: y 2024-04-05T02:54:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:11 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: pgmap v44: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: Reconfiguring daemon grafana.a on smithi196 2024-04-05T02:54:13.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:13 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1394730071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:13.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: pgmap v44: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: Reconfiguring daemon grafana.a on smithi196 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1394730071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: pgmap v44: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 930 op/s 2024-04-05T02:54:13.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: Reconfiguring grafana.a (dependencies changed)... 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: Reconfiguring daemon grafana.a on smithi196 2024-04-05T02:54:13.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1394730071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:13.979 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:54:13 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:54:13.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T02:54:15.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:15 smithi196 ceph-mon[33169]: pgmap v45: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-04-05T02:54:15.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:15 smithi150 ceph-mon[29558]: pgmap v45: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-04-05T02:54:15.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:15 smithi150 ceph-mon[41818]: pgmap v45: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-04-05T02:54:16.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:16 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/873684238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:16.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:16 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/873684238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:16 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/873684238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:17.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:17 smithi196 ceph-mon[33169]: pgmap v46: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-04-05T02:54:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:17 smithi150 ceph-mon[41818]: pgmap v46: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-04-05T02:54:17.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:17 smithi150 ceph-mon[29558]: pgmap v46: 89 pgs: 89 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-04-05T02:54:18.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3975039197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:18.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:54:18 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:54:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:54:18.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3975039197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:18.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3975039197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:19.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:19 smithi150 ceph-mon[41818]: pgmap v47: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-04-05T02:54:19.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:19 smithi150 ceph-mon[29558]: pgmap v47: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-04-05T02:54:19.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:19 smithi196 ceph-mon[33169]: pgmap v47: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 960 op/s 2024-04-05T02:54:20.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:20 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:20.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:20 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:20 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:20.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:54:20] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2024-04-05T02:54:21.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:21 smithi196 ceph-mon[33169]: pgmap v48: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-05T02:54:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/39892050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:21.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:21 smithi150 ceph-mon[41818]: pgmap v48: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-05T02:54:21.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:21 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/39892050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:21.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:21 smithi150 ceph-mon[29558]: pgmap v48: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 657 op/s 2024-04-05T02:54:21.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:21 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/39892050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:22.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:22 smithi196 ceph-mon[33169]: mgrmap e27: x(active, since 91s), standbys: y 2024-04-05T02:54:22.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:22 smithi150 ceph-mon[41818]: mgrmap e27: x(active, since 91s), standbys: y 2024-04-05T02:54:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:22 smithi150 ceph-mon[29558]: mgrmap e27: x(active, since 91s), standbys: y 2024-04-05T02:54:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:23 smithi196 ceph-mon[33169]: pgmap v49: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 1.03k op/s 2024-04-05T02:54:23.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:23 smithi150 ceph-mon[41818]: pgmap v49: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 1.03k op/s 2024-04-05T02:54:23.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:23 smithi150 ceph-mon[29558]: pgmap v49: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 1.03k op/s 2024-04-05T02:54:24.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:24 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1249044001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:24 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1249044001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:24 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1249044001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:25.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:25 smithi196 ceph-mon[33169]: pgmap v50: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-04-05T02:54:25.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:25 smithi150 ceph-mon[41818]: pgmap v50: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-04-05T02:54:25.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:25 smithi150 ceph-mon[29558]: pgmap v50: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-04-05T02:54:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:26 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1168151406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:26.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:26 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1168151406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:26 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1168151406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:27 smithi196 ceph-mon[33169]: pgmap v51: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-04-05T02:54:27.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:27 smithi150 ceph-mon[29558]: pgmap v51: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-04-05T02:54:27.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:27 smithi150 ceph-mon[41818]: pgmap v51: 89 pgs: 89 active+clean; 2.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 255 B/s wr, 725 op/s 2024-04-05T02:54:29.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:54:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:54:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:54:29.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:29 smithi196 ceph-mon[33169]: pgmap v52: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-04-05T02:54:29.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:29 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/860620039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:29.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:29 smithi150 ceph-mon[41818]: pgmap v52: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-04-05T02:54:29.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/860620039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:29.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:29 smithi150 ceph-mon[29558]: pgmap v52: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-04-05T02:54:29.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:29 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/860620039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:30.891 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:54:30] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2024-04-05T02:54:31.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:31 smithi196 ceph-mon[33169]: pgmap v53: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 741 op/s 2024-04-05T02:54:31.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:31 smithi150 ceph-mon[41818]: pgmap v53: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 741 op/s 2024-04-05T02:54:31.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:31 smithi150 ceph-mon[29558]: pgmap v53: 89 pgs: 89 active+clean; 2.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 255 B/s wr, 741 op/s 2024-04-05T02:54:32.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:32 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3939264556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:32 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3939264556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:32.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:32 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3939264556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:33.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:33 smithi196 ceph-mon[33169]: pgmap v54: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-04-05T02:54:33.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:33 smithi150 ceph-mon[41818]: pgmap v54: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-04-05T02:54:33.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:33 smithi150 ceph-mon[29558]: pgmap v54: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 255 B/s wr, 1.10k op/s 2024-04-05T02:54:34.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:34 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1970824124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:34.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:34 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1970824124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:34 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1970824124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:35 smithi196 ceph-mon[33169]: pgmap v55: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-04-05T02:54:35.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:35 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:35.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:35 smithi150 ceph-mon[29558]: pgmap v55: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-04-05T02:54:35.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:35 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:35.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:35 smithi150 ceph-mon[41818]: pgmap v55: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-04-05T02:54:35.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:35 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:37.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:37 smithi196 ceph-mon[33169]: pgmap v56: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-04-05T02:54:37.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:37 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2642047677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:37.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:37 smithi150 ceph-mon[41818]: pgmap v56: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-04-05T02:54:37.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:37 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2642047677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:37.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:37 smithi150 ceph-mon[29558]: pgmap v56: 89 pgs: 89 active+clean; 2.0 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 728 op/s 2024-04-05T02:54:37.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:37 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2642047677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:39.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:54:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:54:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:54:39.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:39 smithi196 ceph-mon[33169]: pgmap v57: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:54:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:39 smithi150 ceph-mon[41818]: pgmap v57: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:54:39.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:39 smithi150 ceph-mon[29558]: pgmap v57: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:54:40.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:40 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4214638840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:40.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:54:40] "GET /metrics HTTP/1.1" 200 36279 "" "Prometheus/2.43.0" 2024-04-05T02:54:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:40 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4214638840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:40 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4214638840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:41 smithi196 ceph-mon[33169]: pgmap v58: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-04-05T02:54:41.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:41 smithi150 ceph-mon[29558]: pgmap v58: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-04-05T02:54:41.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:41 smithi150 ceph-mon[41818]: pgmap v58: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 722 op/s 2024-04-05T02:54:42.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:42 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2800033370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:42 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2800033370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:42 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2800033370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:43 smithi150 ceph-mon[41818]: pgmap v59: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:54:43.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:43 smithi150 ceph-mon[29558]: pgmap v59: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:54:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:43 smithi196 ceph-mon[33169]: pgmap v59: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-04-05T02:54:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:45 smithi196 ceph-mon[33169]: pgmap v60: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-05T02:54:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:45 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/189620715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:45.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:45 smithi150 ceph-mon[41818]: pgmap v60: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-05T02:54:45.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:45 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/189620715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:45.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:45 smithi150 ceph-mon[29558]: pgmap v60: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-05T02:54:45.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:45 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/189620715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:47 smithi150 ceph-mon[29558]: pgmap v61: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-05T02:54:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:47 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1778046284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:47 smithi150 ceph-mon[41818]: pgmap v61: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-05T02:54:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:47 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1778046284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:48.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:47 smithi196 ceph-mon[33169]: pgmap v61: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 678 op/s 2024-04-05T02:54:48.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:47 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1778046284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:48.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:48 smithi196 systemd[1]: Stopping Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:54:49.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:54:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:54:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:54:49.137 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:48 smithi196 podman[78097]: 2024-04-05 02:54:48.835155381 +0000 UTC m=+0.169033205 container died 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-type=git, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner ) 2024-04-05T02:54:49.496 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 podman[78097]: 2024-04-05 02:54:49.27571385 +0000 UTC m=+0.609591678 container cleanup 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, distribution-scope=public, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, vcs-type=git, vendor=Red Hat, Inc., io.buildah.version=1.32.0, architecture=x86_64) 2024-04-05T02:54:49.496 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 bash[78097]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a 2024-04-05T02:54:49.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:49 smithi196 ceph-mon[33169]: pgmap v62: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:54:49.886 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 podman[78114]: 2024-04-05 02:54:49.496895329 +0000 UTC m=+0.662651026 container remove 75718600d44492da7df587c233a4b34673fcc5345d2a5aa4aa804b81c86b0dbe (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, io.buildah.version=1.32.0, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8) 2024-04-05T02:54:49.886 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-05T02:54:49.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:49 smithi150 ceph-mon[41818]: pgmap v62: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:54:49.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:49 smithi150 ceph-mon[29558]: pgmap v62: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:54:50.242 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:54:50.242 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:54:50.242 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 6.912s CPU time. 2024-04-05T02:54:50.242 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:49 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:54:50.556 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 podman[78204]: 2024-04-05 02:54:50.150319609 +0000 UTC m=+0.045441628 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:54:50.557 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 podman[78204]: 2024-04-05 02:54:50.27037393 +0000 UTC m=+0.165495931 container create e13f701bb8c683bc1fa102eaf904eb5792b1771c30d94eb4b4b8eb0470b60f02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.expose-services=, vendor=Red Hat, Inc., io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container) 2024-04-05T02:54:50.557 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 podman[78204]: 2024-04-05 02:54:50.557269201 +0000 UTC m=+0.452391210 container init e13f701bb8c683bc1fa102eaf904eb5792b1771c30d94eb4b4b8eb0470b60f02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, name=ubi8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git) 2024-04-05T02:54:50.835 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:50.836 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3362603104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:50.836 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:54:50] "GET /metrics HTTP/1.1" 200 36284 "" "Prometheus/2.43.0" 2024-04-05T02:54:50.836 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:50.785+0000 7fc8c1015700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-05T02:54:50.836 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 podman[78204]: 2024-04-05 02:54:50.568520453 +0000 UTC m=+0.463642458 container start e13f701bb8c683bc1fa102eaf904eb5792b1771c30d94eb4b4b8eb0470b60f02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, version=8.5, name=ubi8, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, release=239.1651231664, vcs-type=git, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T02:54:50.836 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 bash[78204]: e13f701bb8c683bc1fa102eaf904eb5792b1771c30d94eb4b4b8eb0470b60f02 2024-04-05T02:54:50.836 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:54:50.920 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:50.920 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3362603104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:50.920 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:50] ENGINE Bus STOPPING 2024-04-05T02:54:50.921 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:54:50.921 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:50] ENGINE Bus STOPPED 2024-04-05T02:54:50.921 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:50.921 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3362603104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:51.117 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[78217]: logger=settings t=2024-04-05T02:54:50.885523098Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:54:51.117 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:50 smithi196 podman[78277]: 2024-04-05 02:54:50.967660095 +0000 UTC m=+0.060600783 container died e13f701bb8c683bc1fa102eaf904eb5792b1771c30d94eb4b4b8eb0470b60f02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., name=ubi8, architecture=x86_64, io.buildah.version=1.32.0, release=239.1651231664, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, version=8.5, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-05T02:54:51.117 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ignoring --setuser ceph since I am not root 2024-04-05T02:54:51.118 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ignoring --setgroup ceph since I am not root 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:50] ENGINE Bus STARTING 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.090+0000 7f84169ef700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.090+0000 7f84169ef700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.090+0000 7f84169ef700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.090+0000 7f84169ef700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.090+0000 7f84169ef700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.095+0000 7f841b1f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.095+0000 7f841b1f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.095+0000 7f841b1f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.095+0000 7f841b1f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:54:51.095+0000 7f841b1f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:54:51.229 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:51] ENGINE Serving on http://:::9283 2024-04-05T02:54:51.229 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:54:51] ENGINE Bus STARTED 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: pgmap v63: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T02:54:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: osdmap e77: 8 total, 8 up, 8 in 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.24757 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: mgrmap e28: y(active, starting, since 0.00842388s) 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:54:51.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: Manager daemon y is now available 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:54:51.980 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:54:51.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: pgmap v63: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-04-05T02:54:51.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:51.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:51.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:54:51.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: osdmap e77: 8 total, 8 up, 8 in 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.24757 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: mgrmap e28: y(active, starting, since 0.00842388s) 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:54:51.983 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: Manager daemon y is now available 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:54:51.984 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:51 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:54:52.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: pgmap v63: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2024-04-05T02:54:52.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:52.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' 2024-04-05T02:54:52.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:54:52.003 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 172.21.15.196:0/634217794' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: osdmap e77: 8 total, 8 up, 8 in 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.24757 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: mgrmap e28: y(active, starting, since 0.00842388s) 2024-04-05T02:54:52.004 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:54:52.005 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:54:52.005 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:54:52.005 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:54:52.005 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:54:52.005 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:54:52.005 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:54:52.006 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:54:52.006 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:54:52.006 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:54:52.006 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:54:52.006 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:54:52.006 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: Manager daemon y is now available 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:54:52.007 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:54:52.008 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:54:52.008 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:54:52.008 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:54:52.008 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:51.822+0000 7f7f70cc0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:54:52.008 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:52.001+0000 7f7f70cc0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:54:53.059 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:52 smithi196 ceph-mon[33169]: mgrmap e29: y(active, since 1.00947s) 2024-04-05T02:54:53.059 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:52 smithi196 ceph-mon[33169]: [05/Apr/2024:02:54:51] ENGINE Bus STARTING 2024-04-05T02:54:53.059 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:52 smithi196 ceph-mon[33169]: pgmap v3: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:53.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:52 smithi150 ceph-mon[29558]: mgrmap e29: y(active, since 1.00947s) 2024-04-05T02:54:53.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:52 smithi150 ceph-mon[29558]: [05/Apr/2024:02:54:51] ENGINE Bus STARTING 2024-04-05T02:54:53.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:52 smithi150 ceph-mon[29558]: pgmap v3: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:53.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:52 smithi150 ceph-mon[41818]: mgrmap e29: y(active, since 1.00947s) 2024-04-05T02:54:53.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:52 smithi150 ceph-mon[41818]: [05/Apr/2024:02:54:51] ENGINE Bus STARTING 2024-04-05T02:54:53.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:52 smithi150 ceph-mon[41818]: pgmap v3: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:53.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:53.058+0000 7f7f70cc0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:54:53.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:53.217+0000 7f7f70cc0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:54:53.744 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:53.431+0000 7f7f70cc0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:54:53.744 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:53.600+0000 7f7f70cc0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: [05/Apr/2024:02:54:51] ENGINE Serving on http://172.21.15.150:8765 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: [05/Apr/2024:02:54:52] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: [05/Apr/2024:02:54:52] ENGINE Bus STARTED 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: mgrmap e30: y(active, since 2s) 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: pgmap v4: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1911358297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:54.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:53.743+0000 7f7f70cc0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:54:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: [05/Apr/2024:02:54:51] ENGINE Serving on http://172.21.15.150:8765 2024-04-05T02:54:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: [05/Apr/2024:02:54:52] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:54:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: [05/Apr/2024:02:54:52] ENGINE Bus STARTED 2024-04-05T02:54:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: mgrmap e30: y(active, since 2s) 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: pgmap v4: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1911358297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: [05/Apr/2024:02:54:51] ENGINE Serving on http://172.21.15.150:8765 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: [05/Apr/2024:02:54:52] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: [05/Apr/2024:02:54:52] ENGINE Bus STARTED 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: mgrmap e30: y(active, since 2s) 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: pgmap v4: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1911358297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:53 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:54.459 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:54.189+0000 7f7f70cc0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:54:54.460 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:54.366+0000 7f7f70cc0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:54:54.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:54.526+0000 7f7f70cc0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:54:54.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:54.690+0000 7f7f70cc0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:54:55.217 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:54.962+0000 7f7f70cc0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:54:55.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:55 smithi196 podman[78277]: 2024-04-05 02:54:55.217770913 +0000 UTC m=+4.310711587 container remove e13f701bb8c683bc1fa102eaf904eb5792b1771c30d94eb4b4b8eb0470b60f02 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, version=8.5, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, release=239.1651231664, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.component=ubi8-container, distribution-scope=public, io.buildah.version=1.32.0, name=ubi8, io.openshift.expose-services=) 2024-04-05T02:54:55.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:55 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:54:56.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:55 smithi196 ceph-mon[33169]: pgmap v5: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:56.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:55 smithi196 ceph-mon[33169]: mgrmap e31: y(active, since 4s) 2024-04-05T02:54:56.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:55 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/415004054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:56.137 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:55 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:54:56.137 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:54:55 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.160s CPU time. 2024-04-05T02:54:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:55 smithi150 ceph-mon[29558]: pgmap v5: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:55 smithi150 ceph-mon[29558]: mgrmap e31: y(active, since 4s) 2024-04-05T02:54:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:55 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/415004054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:55 smithi150 ceph-mon[41818]: pgmap v5: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:55 smithi150 ceph-mon[41818]: mgrmap e31: y(active, since 4s) 2024-04-05T02:54:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:55 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/415004054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:56.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:56 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:56.608+0000 7f7f70cc0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:54:56.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:56 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:56.842+0000 7f7f70cc0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:54:57.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:57 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:57 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:57 smithi196 ceph-mon[33169]: pgmap v6: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:57.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:57 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:57.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:57 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:57.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:57 smithi150 ceph-mon[41818]: pgmap v6: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:57.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:57 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:57.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:57 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:57.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:57 smithi150 ceph-mon[29558]: pgmap v6: 89 pgs: 89 active+clean; 986 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-04-05T02:54:57.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:57 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:57.403+0000 7f7f70cc0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:54:57.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:57 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:57.570+0000 7f7f70cc0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:54:58.250 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:57 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:57.932+0000 7f7f70cc0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:54:58.250 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:58.249+0000 7f7f70cc0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:54:58.863 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:58 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:58.863 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:58 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:58.863 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:58 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2815141579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:58.863 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:58 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:58.863 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:58 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:58.863 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:58 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2815141579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:58.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:54:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:54:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:54:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:58 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:58 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:54:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:58 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2815141579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:54:58.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:58.795+0000 7f7f70cc0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:54:59.219 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:59 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:59.216+0000 7f7f70cc0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:54:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:54:59 smithi196 ceph-mon[33169]: pgmap v7: 89 pgs: 89 active+clean; 768 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 514 op/s 2024-04-05T02:54:59.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:54:59 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:54:59.661+0000 7f7f70cc0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:54:59.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:54:59 smithi150 ceph-mon[29558]: pgmap v7: 89 pgs: 89 active+clean; 768 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 514 op/s 2024-04-05T02:54:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:54:59 smithi150 ceph-mon[41818]: pgmap v7: 89 pgs: 89 active+clean; 768 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 514 op/s 2024-04-05T02:55:01.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:01 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:55:01.044+0000 7f7f70cc0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:55:01.751 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:01.751 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:01.752 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[41818]: pgmap v8: 89 pgs: 89 active+clean; 768 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 399 op/s 2024-04-05T02:55:01.752 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3000578588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:01.752 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:01.752 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:01.752 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[29558]: pgmap v8: 89 pgs: 89 active+clean; 768 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 399 op/s 2024-04-05T02:55:01.752 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:01 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3000578588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:01 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:01.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:01 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:01.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:01 smithi196 ceph-mon[33169]: pgmap v8: 89 pgs: 89 active+clean; 768 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 399 op/s 2024-04-05T02:55:01.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:01 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3000578588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:02.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:01 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:55:01.985+0000 7f7f70cc0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:55:02.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:55:02.158+0000 7f7f70cc0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:55:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:55:02.544+0000 7f7f70cc0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:55:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:55:02] ENGINE Bus STARTING 2024-04-05T02:55:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: CherryPy Checker: 2024-04-05T02:55:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: The Application mounted at '' has an empty config. 2024-04-05T02:55:02.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: 2024-04-05T02:55:02.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:55:02] ENGINE Serving on http://:::9283 2024-04-05T02:55:02.888 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:02 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: [05/Apr/2024:02:55:02] ENGINE Bus STARTED 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: Standby manager daemon x started 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:03.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:55:03.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:55:03.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:55:03.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:03 smithi196 ceph-mon[33169]: pgmap v9: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 629 op/s 2024-04-05T02:55:03.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: Standby manager daemon x started 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:55:03.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[41818]: pgmap v9: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 629 op/s 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: Standby manager daemon x started 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/897385257' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:03.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:55:03.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:55:03.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:55:03.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:03 smithi150 ceph-mon[29558]: pgmap v9: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 629 op/s 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: mgrmap e32: y(active, since 12s), standbys: x 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2908226883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:04.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:04 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: mgrmap e32: y(active, since 12s), standbys: x 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2908226883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:04.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: mgrmap e32: y(active, since 12s), standbys: x 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2908226883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:04.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:04 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:55:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: pgmap v10: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 598 op/s 2024-04-05T02:55:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:55:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:05 smithi196 ceph-mon[33169]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:55:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:55:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: pgmap v10: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 598 op/s 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[41818]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: pgmap v10: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 598 op/s 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-05T02:55:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:05 smithi150 ceph-mon[29558]: Reconfiguring daemon prometheus.a on smithi196 2024-04-05T02:55:06.012 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:05 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-04-05T02:55:06.012 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:05 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:06.012 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:05 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.160s CPU time. 2024-04-05T02:55:06.013 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:05 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:06 smithi196 ceph-mon[33169]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T02:55:06.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:06 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:06.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:06 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:06.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:06 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1734533671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:06.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[41818]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T02:55:06.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:06.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1734533671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:06.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[29558]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T02:55:06.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:06.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:06.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:06 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1734533671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:07.042 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:06 smithi196 podman[80217]: 2024-04-05 02:55:06.587982166 +0000 UTC m=+0.030276235 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:55:07.042 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:06 smithi196 podman[80217]: 2024-04-05 02:55:06.721533006 +0000 UTC m=+0.163827096 container create d8e495e7547decea4161da83e32b9f1e9d159b96a515fb41fb7fd59f2546ca3e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, architecture=x86_64) 2024-04-05T02:55:07.042 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:07 smithi196 podman[80217]: 2024-04-05 02:55:07.042326675 +0000 UTC m=+0.484620726 container init d8e495e7547decea4161da83e32b9f1e9d159b96a515fb41fb7fd59f2546ca3e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, distribution-scope=public, name=ubi8, com.redhat.component=ubi8-container, io.openshift.expose-services=, architecture=x86_64, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, io.openshift.tags=base rhel8) 2024-04-05T02:55:07.324 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:07 smithi196 podman[80217]: 2024-04-05 02:55:07.057231195 +0000 UTC m=+0.499525243 container start d8e495e7547decea4161da83e32b9f1e9d159b96a515fb41fb7fd59f2546ca3e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, release=239.1651231664, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, architecture=x86_64, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.buildah.version=1.32.0, distribution-scope=public, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T02:55:07.324 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:07 smithi196 bash[80217]: d8e495e7547decea4161da83e32b9f1e9d159b96a515fb41fb7fd59f2546ca3e 2024-04-05T02:55:07.324 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:07 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:07.628 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:07 smithi150 ceph-mon[41818]: pgmap v11: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 598 op/s 2024-04-05T02:55:07.628 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:07 smithi150 ceph-mon[29558]: pgmap v11: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 598 op/s 2024-04-05T02:55:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:07 smithi196 ceph-mon[33169]: pgmap v11: 89 pgs: 89 active+clean; 561 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 598 op/s 2024-04-05T02:55:07.636 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:07 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[80235]: logger=settings t=2024-04-05T02:55:07.376436529Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:55:08.521 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:08 smithi196 podman[80274]: 2024-04-05 02:55:08.202058708 +0000 UTC m=+0.810919179 container died d8e495e7547decea4161da83e32b9f1e9d159b96a515fb41fb7fd59f2546ca3e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.buildah.version=1.32.0, io.openshift.expose-services=, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, name=ubi8, release=239.1651231664, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T02:55:09.090 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:08 smithi196 podman[80274]: 2024-04-05 02:55:08.839468408 +0000 UTC m=+1.448328852 container remove d8e495e7547decea4161da83e32b9f1e9d159b96a515fb41fb7fd59f2546ca3e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-type=git, release=239.1651231664, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, architecture=x86_64, name=ubi8, vendor=Red Hat, Inc., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, distribution-scope=public, version=8.5) 2024-04-05T02:55:09.090 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:08 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:55:09.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:09 smithi150 ceph-mon[29558]: pgmap v12: 89 pgs: 89 active+clean; 315 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 939 op/s 2024-04-05T02:55:09.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:09 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4036587072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:09.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:09 smithi150 ceph-mon[41818]: pgmap v12: 89 pgs: 89 active+clean; 315 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 939 op/s 2024-04-05T02:55:09.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:09 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4036587072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:09.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:09 smithi196 ceph-mon[33169]: pgmap v12: 89 pgs: 89 active+clean; 315 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 939 op/s 2024-04-05T02:55:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:09 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4036587072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:10.830 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:10 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[68697]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:55:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-05T02:55:11.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:10 smithi196 systemd[1]: Stopping Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:11.137 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:10 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:55:11.137 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:10 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.121s CPU time. 2024-04-05T02:55:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-mon[33169]: pgmap v13: 89 pgs: 89 active+clean; 315 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T02:55:11.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T02:55:11.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.310Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T02:55:11.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.311Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T02:55:11.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.311Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T02:55:11.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.311Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T02:55:11.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[76815]: ts=2024-04-05T02:55:11.311Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T02:55:11.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 podman[80407]: 2024-04-05 02:55:11.321473502 +0000 UTC m=+0.154495306 container died 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:55:11.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:11 smithi150 ceph-mon[41818]: pgmap v13: 89 pgs: 89 active+clean; 315 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2024-04-05T02:55:11.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:11 smithi150 ceph-mon[29558]: pgmap v13: 89 pgs: 89 active+clean; 315 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 628 op/s 2024-04-05T02:55:12.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 podman[80407]: 2024-04-05 02:55:11.926735396 +0000 UTC m=+0.759757199 container cleanup 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:55:12.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:11 smithi196 bash[80407]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T02:55:12.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:12 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2823039298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:12.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 podman[80424]: 2024-04-05 02:55:12.237227136 +0000 UTC m=+0.913589964 container remove 94644404c808936d2409b7ef9d589dea1f4dbff17077a947b8bbd8ba7af45d2d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:55:12.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service: Deactivated successfully. 2024-04-05T02:55:12.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 systemd[1]: Stopped Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:12.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service: Consumed 1.754s CPU time. 2024-04-05T02:55:12.638 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 systemd[1]: Starting Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:12 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/2823039298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:12.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:12 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/2823039298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:12.978 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Removed 128277 objects 2024-04-05T02:55:12.978 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Clean up completed and total clean up time :157.128 2024-04-05T02:55:13.099 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 podman[80506]: 2024-04-05 02:55:12.722587568 +0000 UTC m=+0.043777427 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-05T02:55:13.099 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:12 smithi196 podman[80506]: 2024-04-05 02:55:12.855183332 +0000 UTC m=+0.176373172 container create dc41e7016befea0e504a0f2ebe26d3681427fb329801eeca3f819a2ca3885c47 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:55:13.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 podman[80506]: 2024-04-05 02:55:13.099614005 +0000 UTC m=+0.420803849 container init dc41e7016befea0e504a0f2ebe26d3681427fb329801eeca3f819a2ca3885c47 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:55:13.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 podman[80506]: 2024-04-05 02:55:13.105271366 +0000 UTC m=+0.426461200 container start dc41e7016befea0e504a0f2ebe26d3681427fb329801eeca3f819a2ca3885c47 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T02:55:13.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.128Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-05T02:55:13.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.128Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-05T02:55:13.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.128Z caller=main.go:570 level=info host_details="(Linux 5.14.0-435.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Mar 29 18:46:37 UTC 2024 x86_64 smithi196 (none))" 2024-04-05T02:55:13.387 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.128Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.128Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.130Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.131Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.131Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.131Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.133Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.133Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.553µs 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.133Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.133Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.150Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.152Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.153Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.153Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.748µs wal_replay_duration=19.86783ms wbl_replay_duration=111ns total_replay_duration=19.902704ms 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.155Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-05T02:55:13.388 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.155Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-05T02:55:13.389 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.155Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-05T02:55:13.389 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.175Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.811648ms db_storage=1.238µs remote_storage=1.229µs web_handler=360ns query_engine=579ns scrape=567.963µs scrape_sd=87.24µs notify=21.702µs notify_sd=16.321µs rules=18.884791ms tracing=6.653µs 2024-04-05T02:55:13.389 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.175Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-05T02:55:13.389 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:13.175Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-05T02:55:13.389 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 bash[80506]: dc41e7016befea0e504a0f2ebe26d3681427fb329801eeca3f819a2ca3885c47 2024-04-05T02:55:13.389 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:13 smithi196 systemd[1]: Started Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[29558]: pgmap v14: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[41818]: pgmap v14: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-04-05T02:55:13.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:13.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:13.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:55:13.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:13 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:13.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-mon[33169]: pgmap v14: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 983 op/s 2024-04-05T02:55:13.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:13.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:13.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:55:13.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:13 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:14.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:14 smithi150 ceph-mon[41818]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:55:14.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:14 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/787228191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:14.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:14 smithi150 ceph-mon[29558]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:55:14.731 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:14 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/787228191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:14.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:14 smithi196 ceph-mon[33169]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-05T02:55:14.839 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:14 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/787228191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:14.948 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-04-05T02:55:14.949 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-04-05T02:55:15.219 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:55:15.682 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[29558]: pgmap v15: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-05T02:55:15.682 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:55:15.682 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[41818]: pgmap v15: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:55:15.683 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:15 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:15.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:15 smithi196 ceph-mon[33169]: pgmap v15: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 695 op/s 2024-04-05T02:55:15.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:15 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:55:15.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:15 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:55:15.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:15 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:55:15.767 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:15 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:16.678 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:16 smithi150 ceph-mon[29558]: Upgrade: Updating mgr.x 2024-04-05T02:55:16.678 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:16 smithi150 ceph-mon[29558]: Deploying daemon mgr.x on smithi196 2024-04-05T02:55:16.678 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:16 smithi150 ceph-mon[41818]: Upgrade: Updating mgr.x 2024-04-05T02:55:16.679 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:16 smithi150 ceph-mon[41818]: Deploying daemon mgr.x on smithi196 2024-04-05T02:55:16.689 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:16 smithi196 ceph-mon[33169]: Upgrade: Updating mgr.x 2024-04-05T02:55:16.689 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:16 smithi196 ceph-mon[33169]: Deploying daemon mgr.x on smithi196 2024-04-05T02:55:16.992 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:16 smithi196 systemd[1]: Stopping Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:17.349 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:16 smithi196 podman[80917]: 2024-04-05 02:55:16.993155112 +0000 UTC m=+0.162162302 container died 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T02:55:17.431 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_0' removed 2024-04-05T02:55:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:17 smithi196 ceph-mon[33169]: pgmap v16: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-05T02:55:17.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:17 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3712700361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:17.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:17 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3050186759' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:55:17.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 podman[80917]: 2024-04-05 02:55:17.35041114 +0000 UTC m=+0.519418324 container cleanup 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True) 2024-04-05T02:55:17.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 bash[80917]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x 2024-04-05T02:55:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:17 smithi150 ceph-mon[29558]: pgmap v16: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-05T02:55:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:17 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3712700361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:17.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:17 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3050186759' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:55:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:17 smithi150 ceph-mon[41818]: pgmap v16: 89 pgs: 89 active+clean; 59 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 669 op/s 2024-04-05T02:55:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:17 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3712700361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:17 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3050186759' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:55:18.005 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 podman[80937]: 2024-04-05 02:55:17.660530907 +0000 UTC m=+0.671766168 container remove 4534495b40b00e957bb66c6ffc65053397abb5f813668611f81fea46070bf051 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-05T02:55:18.005 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-05T02:55:18.005 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service: Failed with result 'exit-code'. 2024-04-05T02:55:18.005 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 systemd[1]: Stopped Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:18.005 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:17 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service: Consumed 44.430s CPU time. 2024-04-05T02:55:18.005 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 systemd[1]: Starting Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:18.168 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-05T02:55:18.168 INFO:tasks.radosbench:Beginning radosbench... 2024-04-05T02:55:18.168 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-04-05T02:55:18.169 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool create unique_pool_1 16 2024-04-05T02:55:18.360 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 podman[81072]: 2024-04-05 02:55:18.122077432 +0000 UTC m=+0.018172827 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:55:18.360 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 podman[81072]: 2024-04-05 02:55:18.253945435 +0000 UTC m=+0.150040830 container create 7ca2051bb3927143852d92b48b807b457cd887a349e234ea4a4ffdb5df52af66 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-05T02:55:18.426 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:55:18.724 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:18 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3050186759' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T02:55:18.724 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:18 smithi150 ceph-mon[29558]: osdmap e78: 8 total, 8 up, 8 in 2024-04-05T02:55:18.724 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:18 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3050186759' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T02:55:18.725 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:18 smithi150 ceph-mon[41818]: osdmap e78: 8 total, 8 up, 8 in 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:18 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3050186759' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:18 smithi196 ceph-mon[33169]: osdmap e78: 8 total, 8 up, 8 in 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 podman[81072]: 2024-04-05 02:55:18.481496053 +0000 UTC m=+0.377591448 container init 7ca2051bb3927143852d92b48b807b457cd887a349e234ea4a4ffdb5df52af66 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-0028525) 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 podman[81072]: 2024-04-05 02:55:18.486710271 +0000 UTC m=+0.382805666 container start 7ca2051bb3927143852d92b48b807b457cd887a349e234ea4a4ffdb5df52af66 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, ceph=True) 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 ceph-mgr[81110]: -- 172.21.15.196:0/2651446864 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f5f8d865a0 con 0x55f5f8d5a800 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 bash[81072]: 7ca2051bb3927143852d92b48b807b457cd887a349e234ea4a4ffdb5df52af66 2024-04-05T02:55:18.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:18 smithi196 systemd[1]: Started Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:19.386 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:19.047+0000 7ff65a2ac200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:55:19.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:19.159+0000 7ff65a2ac200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:55:19.831 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:19.831+0000 7ff65a2ac200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:55:19.832 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-mon[33169]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:19.832 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:19.832 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:19.832 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:19.832 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3767296054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:19.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[29558]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:19.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3767296054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[41818]: pgmap v18: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:19 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3767296054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:20.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:19 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:19.925+0000 7ff65a2ac200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:55:20.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.070+0000 7ff65a2ac200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:55:20.539 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.169+0000 7ff65a2ac200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:55:20.539 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.256+0000 7ff65a2ac200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:55:20.539 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.538+0000 7ff65a2ac200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:55:20.752 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_1' created 2024-04-05T02:55:20.816 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3098181839' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-05T02:55:20.816 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.656+0000 7ff65a2ac200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:55:20.816 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.751+0000 7ff65a2ac200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:55:20.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:20 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3098181839' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-05T02:55:20.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:20 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3098181839' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2024-04-05T02:55:21.115 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:20.852+0000 7ff65a2ac200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:55:21.115 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:21.034+0000 7ff65a2ac200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:55:21.376 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-04-05T02:55:21.377 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:21.377 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.121s CPU time. 2024-04-05T02:55:21.377 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:21.460 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-04-05T02:55:21.636 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 podman[81511]: 2024-04-05 02:55:21.279686817 +0000 UTC m=+0.019462775 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:55:21.636 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 podman[81511]: 2024-04-05 02:55:21.382537456 +0000 UTC m=+0.122313398 container create 9b5e36b0b7707fb7c5fb435d07dea3974d461b448171eec18949c5693f540471 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, name=ubi8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-type=git, io.buildah.version=1.32.0, architecture=x86_64, version=8.5, release=239.1651231664, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=) 2024-04-05T02:55:21.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 podman[81511]: 2024-04-05 02:55:21.63718841 +0000 UTC m=+0.376964354 container init 9b5e36b0b7707fb7c5fb435d07dea3974d461b448171eec18949c5693f540471 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8.5, build-date=2022-04-29T11:48:11.055897, name=ubi8, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, architecture=x86_64, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, description=Ceph Grafana Container) 2024-04-05T02:55:21.714 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:55:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:55:21] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2024-04-05T02:55:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-mon[33169]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3098181839' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-05T02:55:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-mon[33169]: osdmap e79: 8 total, 8 up, 8 in 2024-04-05T02:55:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/649595609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-mon[33169]: osdmap e80: 8 total, 8 up, 8 in 2024-04-05T02:55:21.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 podman[81511]: 2024-04-05 02:55:21.642890442 +0000 UTC m=+0.382666385 container start 9b5e36b0b7707fb7c5fb435d07dea3974d461b448171eec18949c5693f540471 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, maintainer=Paul Cuzner , version=8.5, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, release=239.1651231664, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, vendor=Red Hat, Inc.) 2024-04-05T02:55:21.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 bash[81511]: 9b5e36b0b7707fb7c5fb435d07dea3974d461b448171eec18949c5693f540471 2024-04-05T02:55:21.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:21.888 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[81563]: logger=settings t=2024-04-05T02:55:21.806990158Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:55:21.888 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:21 smithi196 podman[81632]: 2024-04-05 02:55:21.841543193 +0000 UTC m=+0.022756145 container died 9b5e36b0b7707fb7c5fb435d07dea3974d461b448171eec18949c5693f540471 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, maintainer=Paul Cuzner , name=ubi8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5) 2024-04-05T02:55:22.088 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[29558]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:22.088 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3098181839' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-05T02:55:22.088 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[29558]: osdmap e79: 8 total, 8 up, 8 in 2024-04-05T02:55:22.088 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:22.088 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/649595609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[29558]: osdmap e80: 8 total, 8 up, 8 in 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[41818]: pgmap v19: 73 pgs: 73 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3098181839' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[41818]: osdmap e79: 8 total, 8 up, 8 in 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/649595609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:22.089 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:21 smithi150 ceph-mon[41818]: osdmap e80: 8 total, 8 up, 8 in 2024-04-05T02:55:22.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:22.101+0000 7ff65a2ac200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:55:22.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:22.236+0000 7ff65a2ac200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:55:22.695 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:22.600+0000 7ff65a2ac200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:55:22.695 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:22.694+0000 7ff65a2ac200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:55:22.695 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:22 smithi196 podman[81632]: 2024-04-05 02:55:22.397514341 +0000 UTC m=+0.578727282 container remove 9b5e36b0b7707fb7c5fb435d07dea3974d461b448171eec18949c5693f540471 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, description=Ceph Grafana Container, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, distribution-scope=public, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=) 2024-04-05T02:55:22.695 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:22 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:55:22.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:22 smithi150 ceph-mon[29558]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:55:22.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:22 smithi150 ceph-mon[29558]: osdmap e81: 8 total, 8 up, 8 in 2024-04-05T02:55:22.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:22 smithi150 ceph-mon[41818]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:55:22.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:22 smithi150 ceph-mon[41818]: osdmap e81: 8 total, 8 up, 8 in 2024-04-05T02:55:23.134 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:22.941+0000 7ff65a2ac200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:55:23.135 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-mon[33169]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:55:23.135 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:22 smithi196 ceph-mon[33169]: osdmap e81: 8 total, 8 up, 8 in 2024-04-05T02:55:23.135 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:22 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:55:23.386 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:23.134+0000 7ff65a2ac200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:55:23.742 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:23.465+0000 7ff65a2ac200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:55:23.742 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:23.741+0000 7ff65a2ac200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:55:23.813 INFO:teuthology.orchestra.run.smithi150.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-04-05T02:55:23.993 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-mon[33169]: pgmap v22: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:23.993 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3664026070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:23.993 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4285803846' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:55:23.993 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:55:24.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[29558]: pgmap v22: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:24.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3664026070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:24.110 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/4285803846' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:55:24.111 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:55:24.111 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[41818]: pgmap v22: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:24.111 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3664026070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:24.111 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4285803846' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:55:24.111 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:23 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:55:24.386 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:23.992+0000 7ff65a2ac200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:55:25.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:24 smithi196 ceph-mon[33169]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:55:25.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:24 smithi196 ceph-mon[33169]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T02:55:25.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:24 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:25.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:24 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:25.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:24.870+0000 7ff65a2ac200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:55:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[41818]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:55:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[41818]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T02:55:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[29558]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:55:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[29558]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T02:55:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:25.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:24 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:25.520 DEBUG:teuthology.orchestra.run.smithi150:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-04-05T02:55:25.523 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-05T02:55:25.523 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-05T02:55:25.574 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:hints = 1 2024-04-05T02:55:25.574 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-05T02:55:25.574 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object prefix: benchmark_data_smithi150_102708 2024-04-05T02:55:25.575 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:55:25.575 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 0 0 0 0 0 0 - 0 2024-04-05T02:55:25.809 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:25.484+0000 7ff65a2ac200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:55:25.809 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:25.611+0000 7ff65a2ac200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:55:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:25 smithi196 ceph-mon[33169]: pgmap v25: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:25 smithi196 ceph-mon[33169]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:55:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:25 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:25 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:26.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:26.050+0000 7ff65a2ac200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:55:26.137 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: [05/Apr/2024:02:55:26] ENGINE Bus STARTING 2024-04-05T02:55:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[29558]: pgmap v25: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[29558]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:55:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[41818]: pgmap v25: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[41818]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:55:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:25 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:26.492 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: CherryPy Checker: 2024-04-05T02:55:26.493 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: The Application mounted at '' has an empty config. 2024-04-05T02:55:26.493 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:26.493 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: [05/Apr/2024:02:55:26] ENGINE Serving on http://:::9283 2024-04-05T02:55:26.493 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: [05/Apr/2024:02:55:26] ENGINE Bus STARTED 2024-04-05T02:55:26.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 1 16 3720 3704 231.248 231.5 0.00472935 0.00431196 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: Standby manager daemon x restarted 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: Standby manager daemon x started 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:26 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3082700889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:27.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: Standby manager daemon x restarted 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: Standby manager daemon x started 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3082700889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: Standby manager daemon x restarted 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: Standby manager daemon x started 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:27.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:27.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: from='mgr.? 172.21.15.196:0/906449656' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:27.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:26 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3082700889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:27.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 2 16 7614 7598 237.298 243.375 0.00366756 0.00420692 2024-04-05T02:55:27.869 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:27 smithi196 ceph-mon[33169]: pgmap v26: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:27.869 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:27 smithi196 ceph-mon[33169]: mgrmap e33: y(active, since 36s), standbys: x 2024-04-05T02:55:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:27 smithi150 ceph-mon[41818]: pgmap v26: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:27 smithi150 ceph-mon[41818]: mgrmap e33: y(active, since 36s), standbys: x 2024-04-05T02:55:28.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:27 smithi150 ceph-mon[29558]: pgmap v26: 89 pgs: 5 creating+activating, 84 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:55:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:27 smithi150 ceph-mon[29558]: mgrmap e33: y(active, since 36s), standbys: x 2024-04-05T02:55:28.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 3 16 11198 11182 232.859 224 0.00300321 0.00428599 2024-04-05T02:55:29.052 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:55:29.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 4 16 14119 14103 220.284 182.562 0.00541863 0.00453284 2024-04-05T02:55:30.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:29 smithi196 ceph-mon[33169]: pgmap v27: 89 pgs: 89 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2024-04-05T02:55:30.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:29 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3991655412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:29 smithi150 ceph-mon[29558]: pgmap v27: 89 pgs: 89 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2024-04-05T02:55:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:29 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/3991655412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:30.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:29 smithi150 ceph-mon[41818]: pgmap v27: 89 pgs: 89 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s wr, 720 op/s 2024-04-05T02:55:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:29 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3991655412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:30.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 5 16 16993 16977 212.151 179.625 0.00563024 0.00470747 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: pgmap v28: 89 pgs: 89 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:55:31.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: Upgrade: Setting container_image for all mgr 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-05T02:55:31.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:30 smithi196 ceph-mon[33169]: Upgrade: It appears safe to stop mon.a 2024-04-05T02:55:31.163 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: pgmap v28: 89 pgs: 89 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: Upgrade: Setting container_image for all mgr 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T02:55:31.164 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[41818]: Upgrade: It appears safe to stop mon.a 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: pgmap v28: 89 pgs: 89 active+clean; 298 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s wr, 628 op/s 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: Upgrade: Setting container_image for all mgr 2024-04-05T02:55:31.165 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-05T02:55:31.166 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:30 smithi150 ceph-mon[29558]: Upgrade: It appears safe to stop mon.a 2024-04-05T02:55:31.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 6 16 19644 19628 204.406 165.688 0.0068315 0.00488555 2024-04-05T02:55:31.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:55:31] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2024-04-05T02:55:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:31 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:31 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:31 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:32.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 7 16 22117 22101 197.284 154.562 0.00876833 0.00506305 2024-04-05T02:55:33.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: pgmap v29: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-04-05T02:55:33.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/46219305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: Upgrade: Updating mon.a 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: pgmap v29: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/46219305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: Upgrade: Updating mon.a 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:55:33.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:32 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: pgmap v29: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/46219305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: Upgrade: Updating mon.a 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='mgr.y' 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:55:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:32 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:55:33.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-04-05T02:55:33.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:33.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:33.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 8 16 24611 24595 192.107 155.875 0.00728239 0.00519925 2024-04-05T02:55:33.765 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 podman[84654]: 2024-04-05 02:55:33.336407753 +0000 UTC m=+0.028018529 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:55:33.766 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 podman[84654]: 2024-04-05 02:55:33.477689194 +0000 UTC m=+0.169299946 container create 1959e2dc9902cdb39c6592d26808046af37e55b844a73dd90313ae163f1c4a9f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, build-date=2022-04-29T11:48:11.055897, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, distribution-scope=public, io.buildah.version=1.32.0, version=8.5, io.openshift.expose-services=, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-05T02:55:34.033 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:33 smithi196 ceph-mon[33169]: Deploying daemon mon.a on smithi150 2024-04-05T02:55:34.033 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 podman[84654]: 2024-04-05 02:55:33.763981079 +0000 UTC m=+0.455591843 container init 1959e2dc9902cdb39c6592d26808046af37e55b844a73dd90313ae163f1c4a9f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, version=8.5, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.openshift.tags=base rhel8, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897) 2024-04-05T02:55:34.033 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 podman[84654]: 2024-04-05 02:55:33.774657313 +0000 UTC m=+0.466268069 container start 1959e2dc9902cdb39c6592d26808046af37e55b844a73dd90313ae163f1c4a9f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, description=Ceph Grafana Container, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, name=ubi8, vcs-type=git, maintainer=Paul Cuzner , distribution-scope=public) 2024-04-05T02:55:34.034 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 bash[84654]: 1959e2dc9902cdb39c6592d26808046af37e55b844a73dd90313ae163f1c4a9f 2024-04-05T02:55:34.034 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:33 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:34.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:33 smithi150 ceph-mon[41818]: Deploying daemon mon.a on smithi150 2024-04-05T02:55:34.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:33 smithi150 ceph-mon[29558]: Deploying daemon mon.a on smithi150 2024-04-05T02:55:34.385 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[84668]: logger=settings t=2024-04-05T02:55:34.032954443Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:55:34.385 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:34 smithi196 podman[84705]: 2024-04-05 02:55:34.077727393 +0000 UTC m=+0.033438030 container died 1959e2dc9902cdb39c6592d26808046af37e55b844a73dd90313ae163f1c4a9f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, build-date=2022-04-29T11:48:11.055897) 2024-04-05T02:55:34.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 9 16 26721 26705 185.414 131.875 0.00418963 0.00538728 2024-04-05T02:55:34.956 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:34 smithi196 podman[84705]: 2024-04-05 02:55:34.686032743 +0000 UTC m=+0.641743370 container remove 1959e2dc9902cdb39c6592d26808046af37e55b844a73dd90313ae163f1c4a9f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.expose-services=, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., name=ubi8, maintainer=Paul Cuzner , io.buildah.version=1.32.0, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, distribution-scope=public, version=8.5) 2024-04-05T02:55:34.957 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:34 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:55:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:34 smithi150 ceph-mon[29558]: pgmap v30: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-05T02:55:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:34 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/654980864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:34 smithi150 ceph-mon[41818]: pgmap v30: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-05T02:55:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:34 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/654980864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:35.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:34 smithi196 ceph-mon[33169]: pgmap v30: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 125 MiB/s wr, 2.01k op/s 2024-04-05T02:55:35.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:34 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/654980864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:35.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:35 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:55:35.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:35 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.048s CPU time. 2024-04-05T02:55:35.576 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 10 16 28736 28720 179.466 125.938 0.00310099 0.00556745 2024-04-05T02:55:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:35 smithi150 ceph-mon[29558]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:36.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:35 smithi150 ceph-mon[41818]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:35 smithi196 ceph-mon[33169]: from='mgr.15156 172.21.15.150:0/1180020068' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:36.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 11 16 30844 30828 175.127 131.75 0.00774241 0.00570503 2024-04-05T02:55:37.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:36 smithi150 ceph-mon[29558]: pgmap v31: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-04-05T02:55:37.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:36 smithi150 ceph-mon[41818]: pgmap v31: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-04-05T02:55:37.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:36 smithi196 ceph-mon[33169]: pgmap v31: 89 pgs: 89 active+clean; 1.2 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 110 MiB/s wr, 1.75k op/s 2024-04-05T02:55:37.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 12 15 32827 32812 170.865 124 0.0143044 0.00584628 2024-04-05T02:55:37.938 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:37 smithi150 systemd[1]: Stopping Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:38.196 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:37 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1135404435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:38.196 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:37 smithi150 ceph-mon[29558]: from='client.? 172.21.15.150:0/1135404435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:38.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:37 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1135404435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:38.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 13 16 34619 34603 166.332 111.938 0.0113897 0.00600597 2024-04-05T02:55:39.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:55:39.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:38 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[29535]: 2024-04-05T02:55:38.832+0000 7f634627e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T02:55:39.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:38 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[29535]: 2024-04-05T02:55:38.832+0000 7f634627e700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-05T02:55:39.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:38 smithi150 podman[103239]: 2024-04-05 02:55:38.897058881 +0000 UTC m=+1.106196552 container died ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5) 2024-04-05T02:55:39.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 14 16 36302 36286 161.964 105.188 0.00372686 0.00616679 2024-04-05T02:55:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:40 smithi150 podman[103239]: 2024-04-05 02:55:40.236351098 +0000 UTC m=+2.445488764 container cleanup ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-05T02:55:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:40 smithi150 bash[103239]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a 2024-04-05T02:55:40.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 15 16 38131 38115 158.787 114.312 0.00531103 0.00629192 2024-04-05T02:55:40.756 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:40 smithi150 podman[103265]: 2024-04-05 02:55:40.488571136 +0000 UTC m=+1.585065876 container remove ff4c949de5c044915f9f513186cf47aa85567dd1987d681d728f189bbcdbc021 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T02:55:41.107 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:40 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-05T02:55:41.108 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:40 smithi150 systemd[1]: Stopped Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:41.108 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:40 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service: Consumed 14.817s CPU time. 2024-04-05T02:55:41.442 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 systemd[1]: Starting Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:41.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 16 16 40039 40023 156.315 119.25 0.00783034 0.00639212 2024-04-05T02:55:41.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:55:41] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2024-04-05T02:55:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 podman[103409]: 2024-04-05 02:55:41.347836813 +0000 UTC m=+0.048071255 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:55:41.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 podman[103409]: 2024-04-05 02:55:41.506614458 +0000 UTC m=+0.206848877 container create 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525) 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 podman[103409]: 2024-04-05 02:55:41.810514938 +0000 UTC m=+0.510749365 container init 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 podman[103409]: 2024-04-05 02:55:41.819465419 +0000 UTC m=+0.519699838 container start 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev), process ceph-mon, pid 7 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: pidfile_write: ignore empty --pid-file 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: load: jerasure load: lrc 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: RocksDB version: 7.9.2 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Git sha 0 2024-04-05T02:55:42.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Compile date 2024-04-04 02:01:22 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: DB SUMMARY 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: DB Session ID: OW4PMPGOPVVRUAPAWQ98 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1167 Bytes 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 2283158 ; 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.env: 0x5579cad4bc60 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.fs: PosixFileSystem 2024-04-05T02:55:42.232 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.info_log: 0x5579ccc53ae0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.statistics: (nil) 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.use_fsync: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.db_log_dir: 2024-04-05T02:55:42.233 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.wal_dir: 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.write_buffer_manager: 0x5579ccc26820 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:55:42.234 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.unordered_write: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.row_cache: None 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.wal_filter: None 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.wal_compression: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:55:42.235 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:55:42.236 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_open_files: -1 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Compression algorithms supported: 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:55:42.237 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kZSTD supported: 0 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.merge_operator: 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_filter: None 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:55:42.238 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5579ccc53be0) 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: index_type: 0 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: data_block_index_type: 0 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: index_shortening: 1 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: checksum: 4 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: no_block_cache: 0 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache: 0x5579ccc57090 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache_options: 2024-04-05T02:55:42.239 INFO:journalctl@ceph.mon.a.smithi150.stdout: capacity : 536870912 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: num_shard_bits : 4 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: strict_capacity_limit : 0 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_cache_compressed: (nil) 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: persistent_cache: (nil) 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_size: 4096 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_size_deviation: 10 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_restart_interval: 16 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: index_block_restart_interval: 1 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: metadata_block_size: 4096 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: partition_filters: 0 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: use_delta_encoding: 1 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: filter_policy: bloomfilter 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: whole_key_filtering: 1 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: verify_compression: 0 2024-04-05T02:55:42.240 INFO:journalctl@ceph.mon.a.smithi150.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: format_version: 5 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: enable_index_compression: 1 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: block_align: 0 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: max_auto_readahead_size: 262144 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: prepopulate_block_cache: 0 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: initial_auto_readahead_size: 8192 2024-04-05T02:55:42.241 INFO:journalctl@ceph.mon.a.smithi150.stdout: num_file_reads_for_auto_readahead: 2 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression: NoCompression 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.num_levels: 7 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:55:42.242 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-05T02:55:42.243 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:55:42.244 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.arena_block_size: 1048576 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:55:42.245 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:55:42.246 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.ttl: 2592000 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.blob_file_starting_level: 0 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-04-05T02:55:42.247 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12081, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 2a890589-2981-4376-ab87-9c478357a40d 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285741876614, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285741886753, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 2117370, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12083, "largest_seqno": 13084, "table_properties": {"data_size": 2111847, "index_size": 3288, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10878, "raw_average_key_size": 23, "raw_value_size": 2101592, "raw_average_value_size": 4490, "num_data_blocks": 150, "num_entries": 468, "num_filter_entries": 468, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712285741, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "2a890589-2981-4376-ab87-9c478357a40d", "db_session_id": "OW4PMPGOPVVRUAPAWQ98", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285741891059, "job": 1, "event": "recovery_finished"} 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5579ccd44000 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: DB pointer 0x5579ccd32000 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** DB Stats ** 2024-04-05T02:55:42.248 INFO:journalctl@ceph.mon.a.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: L0 1/0 2.02 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 389.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: L6 1/0 6.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Sum 2/0 8.37 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 389.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 389.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:55:42.249 INFO:journalctl@ceph.mon.a.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 389.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Cumulative compaction: 0.00 GB write, 81.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Interval compaction: 0.00 GB write, 81.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:55:42.250 INFO:journalctl@ceph.mon.a.smithi150.stdout: Block cache BinnedLRUCache@0x5579ccc57090#7 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: starting mon.a rank 0 at public addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] at bind addrs [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???) e3 preinit fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).mds e9 new map 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).mds e9 print_map 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: e9 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: legacy client fscid: 1 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: Filesystem 'foo' (1) 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: fs_name foo 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: epoch 8 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-05T02:55:42.251 INFO:journalctl@ceph.mon.a.smithi150.stdout: created 2024-04-05T02:49:47.190260+0000 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: modified 2024-04-05T02:50:03.911335+0000 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: tableserver 0 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: root 0 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: session_timeout 60 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: session_autoclose 300 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: max_file_size 1099511627776 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: max_xattr_size 65536 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: required_client_features {} 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: last_failure 0 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: last_failure_osd_epoch 0 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: max_mds 1 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: in 0 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: up {0=24557} 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: failed 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: damaged 2024-04-05T02:55:42.252 INFO:journalctl@ceph.mon.a.smithi150.stdout: stopped 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: data_pools [4] 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: metadata_pool 3 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: inline_data disabled 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: balancer 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: bal_rank_mask -1 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: standby_count_wanted 1 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: qdb_cluster leader: 0 members: 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: [mds.foo.smithi150.jnrwgs{0:24557} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: Standby daemons: 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout: [mds.foo.smithi196.eohomm{-1:24526} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).osd e82 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:55:42.253 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).osd e82 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:55:42.254 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 ceph-mon[103445]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-04-05T02:55:42.254 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 bash[103409]: 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d 2024-04-05T02:55:42.254 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:41 smithi150 systemd[1]: Started Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:42.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 17 16 41923 41907 154.046 117.75 0.00925121 0.00648632 2024-04-05T02:55:43.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 18 16 43699 43683 151.654 111 0.0159399 0.00658851 2024-04-05T02:55:43.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ignoring --setuser ceph since I am not root 2024-04-05T02:55:43.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ignoring --setgroup ceph since I am not root 2024-04-05T02:55:43.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mgr[92760]: -- 172.21.15.150:0/2774491202 <== mon.2 v2:172.21.15.150:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55cca0b385a0 con 0x55cca0b0c800 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: pgmap v32: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 170 MiB/s wr, 2.72k op/s 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: pgmap v33: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/451577032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 186 MiB/s wr, 2.98k op/s 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: mon.a calling monitor election 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4189685057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: monmap epoch 3 2024-04-05T02:55:43.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: last_changed 2024-04-05T02:44:29.726178+0000 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: min_mon_release 17 (quincy) 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: election_strategy: 1 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: mgrmap e33: y(active, since 52s), standbys: x 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: daemon grafana.a on smithi196 is in unknown state 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: from='mgr.15156 ' entity='' 2024-04-05T02:55:43.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[103445]: mgrmap e34: y(active, since 52s), standbys: x 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: pgmap v32: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 170 MiB/s wr, 2.72k op/s 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: pgmap v33: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/451577032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 186 MiB/s wr, 2.98k op/s 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: mon.a calling monitor election 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/4189685057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: monmap epoch 3 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: last_changed 2024-04-05T02:44:29.726178+0000 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: min_mon_release 17 (quincy) 2024-04-05T02:55:43.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: election_strategy: 1 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: mgrmap e33: y(active, since 52s), standbys: x 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: daemon grafana.a on smithi196 is in unknown state 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: from='mgr.15156 ' entity='' 2024-04-05T02:55:43.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:43 smithi150 ceph-mon[41818]: mgrmap e34: y(active, since 52s), standbys: x 2024-04-05T02:55:43.886 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: ignoring --setuser ceph since I am not root 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: ignoring --setgroup ceph since I am not root 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mgr[81110]: -- 172.21.15.196:0/690405037 <== mon.1 v2:172.21.15.196:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5580c5a325a0 con 0x5580c5a06800 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: pgmap v32: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 170 MiB/s wr, 2.72k op/s 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: pgmap v33: 89 pgs: 89 active+clean; 1.9 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 144 MiB/s wr, 2.31k op/s 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/451577032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: pgmap v34: 89 pgs: 89 active+clean; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 186 MiB/s wr, 2.98k op/s 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: mon.a calling monitor election 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4189685057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:43.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: monmap epoch 3 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: last_changed 2024-04-05T02:44:29.726178+0000 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: min_mon_release 17 (quincy) 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: election_strategy: 1 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: osdmap e82: 8 total, 8 up, 8 in 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: mgrmap e33: y(active, since 52s), standbys: x 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: daemon grafana.a on smithi196 is in unknown state 2024-04-05T02:55:43.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: from='mgr.15156 ' entity='' 2024-04-05T02:55:43.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:43 smithi196 ceph-mon[33169]: mgrmap e34: y(active, since 52s), standbys: x 2024-04-05T02:55:44.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 19 16 45437 45421 149.389 108.625 0.00966213 0.00668824 2024-04-05T02:55:44.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:44.250+0000 7f62c72ce200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:55:44.637 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:44.399+0000 7f62c72ce200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:55:44.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:44.384+0000 7f46118ae200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-05T02:55:44.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:44.552+0000 7f46118ae200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-05T02:55:45.426 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-04-05T02:55:45.426 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:45.426 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.048s CPU time. 2024-04-05T02:55:45.427 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:45.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:55:45.578868+0000 min lat: 0.00125779 max lat: 0.0973237 avg lat: 0.0068011 2024-04-05T02:55:45.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:55:45.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 20 16 47029 47013 146.894 99.5 0.00775225 0.0068011 2024-04-05T02:55:45.757 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:45.432+0000 7f62c72ce200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:55:45.757 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:45.577+0000 7f62c72ce200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:55:45.757 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 podman[84830]: 2024-04-05 02:55:45.330782702 +0000 UTC m=+0.027720044 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:55:45.757 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 podman[84830]: 2024-04-05 02:55:45.471278056 +0000 UTC m=+0.168215378 container create 29a55f56e64b9acb68aa92cdaa25209ce729fe7cfce636068bf75e40bff0b412 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, description=Ceph Grafana Container, architecture=x86_64, version=8.5, io.openshift.expose-services=, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-04-05T02:55:45.757 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 podman[84830]: 2024-04-05 02:55:45.756530392 +0000 UTC m=+0.453467721 container init 29a55f56e64b9acb68aa92cdaa25209ce729fe7cfce636068bf75e40bff0b412 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.tags=base rhel8, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.openshift.expose-services=, com.redhat.component=ubi8-container, distribution-scope=public, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vcs-type=git, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-04-05T02:55:45.877 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 1 2024-04-05T02:55:45.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:45 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/1207450834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:45.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:45 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1207450834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:45.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:45 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:45.723+0000 7f46118ae200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-05T02:55:45.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:45 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:45.895+0000 7f46118ae200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-05T02:55:46.015 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:45 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/1207450834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:55:46.015 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:45.795+0000 7f62c72ce200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:55:46.015 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:45.946+0000 7f62c72ce200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:55:46.015 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 podman[84830]: 2024-04-05 02:55:45.772054419 +0000 UTC m=+0.468991742 container start 29a55f56e64b9acb68aa92cdaa25209ce729fe7cfce636068bf75e40bff0b412 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, vcs-type=git, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.buildah.version=1.32.0) 2024-04-05T02:55:46.016 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 bash[84830]: 29a55f56e64b9acb68aa92cdaa25209ce729fe7cfce636068bf75e40bff0b412 2024-04-05T02:55:46.016 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:45 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:46.369 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:46.090+0000 7f62c72ce200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:55:46.370 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[84845]: logger=settings t=2024-04-05T02:55:46.015721358Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:55:46.370 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:46 smithi196 podman[84884]: 2024-04-05 02:55:46.06855394 +0000 UTC m=+0.036139200 container died 29a55f56e64b9acb68aa92cdaa25209ce729fe7cfce636068bf75e40bff0b412 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, maintainer=Paul Cuzner , release=239.1651231664, version=8.5, description=Ceph Grafana Container, vendor=Red Hat, Inc., architecture=x86_64, vcs-type=git, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, name=ubi8, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-04-05T02:55:46.460 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:46.136+0000 7f46118ae200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-05T02:55:46.460 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:46.310+0000 7f46118ae200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-05T02:55:46.460 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:46.458+0000 7f46118ae200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-05T02:55:46.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 21 16 48587 48571 144.536 97.375 0.0115526 0.00691296 2024-04-05T02:55:46.636 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:46.504+0000 7f62c72ce200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:55:46.709 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T02:55:46.981 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:46.681+0000 7f62c72ce200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:55:46.981 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:46.830+0000 7f62c72ce200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:55:46.982 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:46 smithi196 podman[84884]: 2024-04-05 02:55:46.662311828 +0000 UTC m=+0.629897082 container remove 29a55f56e64b9acb68aa92cdaa25209ce729fe7cfce636068bf75e40bff0b412 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.buildah.version=1.32.0, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, architecture=x86_64, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, maintainer=Paul Cuzner , vcs-type=git, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8) 2024-04-05T02:55:46.982 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:46 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:55:47.120 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:46 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:46.920+0000 7f46118ae200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-05T02:55:47.120 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:47.118+0000 7f46118ae200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-05T02:55:47.240 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:55:47.243 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:47 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:55:47.243 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:47 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.028s CPU time. 2024-04-05T02:55:47.243 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:46.980+0000 7f62c72ce200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:55:47.243 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:47.242+0000 7f62c72ce200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:55:47.383 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:47.297+0000 7f46118ae200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-05T02:55:47.577 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 22 16 50059 50043 142.148 92 0.0104919 0.00702877 2024-04-05T02:55:47.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:47.470+0000 7f46118ae200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-05T02:55:48.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:47.782+0000 7f46118ae200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-05T02:55:48.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 23 16 51572 51556 140.079 94.5625 0.0123097 0.00713338 2024-04-05T02:55:48.923 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:48.722+0000 7f62c72ce200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:55:48.923 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:55:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:55:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:55:49.386 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:48.923+0000 7f62c72ce200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:55:49.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 24 16 53038 53022 138.06 91.625 0.00560711 0.00723703 2024-04-05T02:55:49.720 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:49.489+0000 7f46118ae200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-05T02:55:49.864 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:49.425+0000 7f62c72ce200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:55:49.865 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:49.565+0000 7f62c72ce200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:55:49.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:49.718+0000 7f46118ae200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-05T02:55:50.139 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:49.864+0000 7f62c72ce200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:55:50.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 25 16 54539 54523 136.289 93.8125 0.00625188 0.00733263 2024-04-05T02:55:50.628 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:50.165+0000 7f62c72ce200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:55:50.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:50.281+0000 7f46118ae200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-05T02:55:50.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:50.447+0000 7f46118ae200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-05T02:55:51.129 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:50.796+0000 7f46118ae200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:55:51.385 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:50.627+0000 7f62c72ce200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:55:51.385 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:51.012+0000 7f62c72ce200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:55:51.385 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:51.384+0000 7f62c72ce200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:55:51.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:51.127+0000 7f46118ae200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-05T02:55:51.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 26 16 56061 56045 134.706 95.125 0.0113772 0.00741876 2024-04-05T02:55:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:51.636+0000 7f46118ae200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-05T02:55:52.327 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:52.062+0000 7f46118ae200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-05T02:55:52.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 27 16 57573 57557 133.217 94.5 0.0207175 0.00750142 2024-04-05T02:55:52.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:52.489+0000 7f46118ae200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-05T02:55:52.728 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:55:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:55:52.327Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T02:55:52.729 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:55:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:55:52.327Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T02:55:53.136 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:52.690+0000 7f62c72ce200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:55:53.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 28 16 58966 58950 131.567 87.0625 0.0119402 0.00759357 2024-04-05T02:55:53.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:53.532+0000 7f62c72ce200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:55:53.887 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:53.686+0000 7f62c72ce200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:55:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[41818]: Standby manager daemon x restarted 2024-04-05T02:55:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[41818]: Standby manager daemon x started 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[41818]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:53.974+0000 7f46118ae200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[103445]: Standby manager daemon x restarted 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[103445]: Standby manager daemon x started 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[103445]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:54.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[103445]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:54.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[103445]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:54.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-mon[103445]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-mon[33169]: Standby manager daemon x restarted 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-mon[33169]: Standby manager daemon x started 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-mon[33169]: from='mgr.? 172.21.15.196:0/4145210594' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:54.026+0000 7f62c72ce200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: [05/Apr/2024:02:55:54] ENGINE Bus STARTING 2024-04-05T02:55:54.387 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: CherryPy Checker: 2024-04-05T02:55:54.388 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: The Application mounted at '' has an empty config. 2024-04-05T02:55:54.388 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: 2024-04-05T02:55:54.388 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: [05/Apr/2024:02:55:54] ENGINE Serving on http://:::9283 2024-04-05T02:55:54.388 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 02:55:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x[81087]: [05/Apr/2024:02:55:54] ENGINE Bus STARTED 2024-04-05T02:55:54.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 29 16 60375 60359 130.067 88.0625 0.0121303 0.00768332 2024-04-05T02:55:54.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:55:54.459Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2024-04-05T02:55:55.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-mon[41818]: mgrmap e35: y(active, since 63s), standbys: x 2024-04-05T02:55:55.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:54.936+0000 7f46118ae200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-05T02:55:55.228 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.108+0000 7f46118ae200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-05T02:55:55.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-mon[103445]: mgrmap e35: y(active, since 63s), standbys: x 2024-04-05T02:55:55.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:55 smithi196 ceph-mon[33169]: mgrmap e35: y(active, since 63s), standbys: x 2024-04-05T02:55:55.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 30 16 61747 61731 128.589 85.75 0.00697914 0.00777209 2024-04-05T02:55:55.812 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:55 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b[33146]: 2024-04-05T02:55:55.501+0000 7f209ea50700 -1 mon.b@1(peon).osd e83 update_from_paxos full map CRC mismatch, resetting to canonical 2024-04-05T02:55:55.813 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c[41795]: 2024-04-05T02:55:55.501+0000 7f2ff3d4e700 -1 mon.c@2(peon).osd e83 update_from_paxos full map CRC mismatch, resetting to canonical 2024-04-05T02:55:55.814 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.487+0000 7f46118ae200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-05T02:55:55.814 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:55:55] ENGINE Bus STARTING 2024-04-05T02:55:56.104 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: CherryPy Checker: 2024-04-05T02:55:56.104 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: The Application mounted at '' has an empty config. 2024-04-05T02:55:56.104 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:56.104 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:55:55] ENGINE Serving on http://:::9283 2024-04-05T02:55:56.104 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:55:55] ENGINE Bus STARTED 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.904+0000 7f4571c87700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.904+0000 7f4571c87700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.904+0000 7f4571c87700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.904+0000 7f4571c87700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.904+0000 7f4571c87700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.909+0000 7f4579cd7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.909+0000 7f4579cd7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.909+0000 7f4579cd7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.909+0000 7f4579cd7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.105 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:55.909+0000 7f4579cd7700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-05T02:55:56.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: Active manager daemon y restarted 2024-04-05T02:55:56.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: Activating manager daemon y 2024-04-05T02:55:56.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: failed to encode map e83 with expected crc 2024-04-05T02:55:56.389 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: mgrmap e36: y(active, starting, since 0.0129848s), standbys: x 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:55:56.390 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: Manager daemon y is now available 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:55:56.391 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:56 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:55:56.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: Active manager daemon y restarted 2024-04-05T02:55:56.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: Activating manager daemon y 2024-04-05T02:55:56.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: failed to encode map e83 with expected crc 2024-04-05T02:55:56.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: mgrmap e36: y(active, starting, since 0.0129848s), standbys: x 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:55:56.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: Manager daemon y is now available 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:55:56.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: Active manager daemon y restarted 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: Activating manager daemon y 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: failed to encode map e83 with expected crc 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: mgrmap e36: y(active, starting, since 0.0129848s), standbys: x 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi196.eohomm"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi150.jnrwgs"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T02:55:56.481 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: Manager daemon y is now available 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:55:56.482 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-05T02:55:56.563 INFO:teuthology.orchestra.run.smithi150.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T02:55:56.563 INFO:teuthology.orchestra.run.smithi150.stdout:alertmanager.a smithi150 *:9093,9094 running (2m) 60s ago 7m 19.6M - 0.25.0 c8568f914cd2 5c82bec44fb9 2024-04-05T02:55:56.563 INFO:teuthology.orchestra.run.smithi150.stdout:grafana.a smithi196 *:3000 unknown 32s ago 6m - - 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi150.jnrwgs smithi150 running (5m) 60s ago 5m 14.7M - 17.2.7-805-g6fda41ff a49113eacc58 cfa13d4de7a1 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi196.eohomm smithi196 running (5m) 32s ago 5m 16.1M - 17.2.7-805-g6fda41ff a49113eacc58 e3ce4d99c3da 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.x smithi196 *:8443,9283,8765 running (38s) 32s ago 11m 65.2M - 19.0.0-2013-g0028525c d8c35eaf3c25 7ca2051bb392 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.y smithi150 *:8443,9283,8765 running (118s) 60s ago 12m 480M - 19.0.0-2013-g0028525c d8c35eaf3c25 0d5b82cc2eb2 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mon.a smithi150 running (13m) 60s ago 13m 55.2M 2048M 17.2.7-805-g6fda41ff a49113eacc58 ff4c949de5c0 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mon.b smithi196 running (11m) 32s ago 11m 44.4M 2048M 17.2.7-805-g6fda41ff a49113eacc58 6680dc817949 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:mon.c smithi150 running (11m) 60s ago 11m 42.3M 2048M 17.2.7-805-g6fda41ff a49113eacc58 826b3558df5d 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.a smithi150 *:9100 running (2m) 60s ago 6m 8334k - 1.5.0 0da6a335fe13 dc5e73ff394f 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.b smithi196 *:9100 running (2m) 32s ago 6m 8162k - 1.5.0 0da6a335fe13 0d264938c335 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:osd.0 smithi150 running (10m) 60s ago 10m 372M 2480M 17.2.7-805-g6fda41ff a49113eacc58 a6b8a0140540 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:osd.1 smithi150 running (10m) 60s ago 10m 245M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2c503b0a41b4 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:osd.2 smithi150 running (10m) 60s ago 10m 309M 2480M 17.2.7-805-g6fda41ff a49113eacc58 fee7e3893826 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:osd.3 smithi150 running (9m) 60s ago 9m 167M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5d887a680c09 2024-04-05T02:55:56.564 INFO:teuthology.orchestra.run.smithi150.stdout:osd.4 smithi196 running (9m) 32s ago 9m 285M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b21803209415 2024-04-05T02:55:56.565 INFO:teuthology.orchestra.run.smithi150.stdout:osd.5 smithi196 running (8m) 32s ago 8m 301M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5415e890a870 2024-04-05T02:55:56.565 INFO:teuthology.orchestra.run.smithi150.stdout:osd.6 smithi196 running (8m) 32s ago 8m 173M 2480M 17.2.7-805-g6fda41ff a49113eacc58 85a4b25b62cf 2024-04-05T02:55:56.565 INFO:teuthology.orchestra.run.smithi150.stdout:osd.7 smithi196 running (8m) 32s ago 8m 205M 2480M 17.2.7-805-g6fda41ff a49113eacc58 ed60a8dad4f7 2024-04-05T02:55:56.565 INFO:teuthology.orchestra.run.smithi150.stdout:prometheus.a smithi196 *:9095 running (43s) 32s ago 7m 43.3M - 2.43.0 a07b618ecd1d dc41e7016bef 2024-04-05T02:55:56.578 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 31 16 62890 62874 126.746 71.4375 0.00689761 0.00788306 2024-04-05T02:55:56.944 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:55:56 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:55:56.536+0000 7f45dfa59700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-05T02:55:57.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-mon[41818]: failed to encode map e83 with expected crc 2024-04-05T02:55:57.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-mon[41818]: [05/Apr/2024:02:55:56] ENGINE Bus STARTING 2024-04-05T02:55:57.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-mon[41818]: mgrmap e37: y(active, since 1.02129s), standbys: x 2024-04-05T02:55:57.228 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T02:55:57.026Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2024-04-05T02:55:57.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-mon[103445]: failed to encode map e83 with expected crc 2024-04-05T02:55:57.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-mon[103445]: [05/Apr/2024:02:55:56] ENGINE Bus STARTING 2024-04-05T02:55:57.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:57 smithi150 ceph-mon[103445]: mgrmap e37: y(active, since 1.02129s), standbys: x 2024-04-05T02:55:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:57 smithi196 ceph-mon[33169]: failed to encode map e83 with expected crc 2024-04-05T02:55:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:57 smithi196 ceph-mon[33169]: [05/Apr/2024:02:55:56] ENGINE Bus STARTING 2024-04-05T02:55:57.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:57 smithi196 ceph-mon[33169]: mgrmap e37: y(active, since 1.02129s), standbys: x 2024-04-05T02:55:57.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:57 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-04-05T02:55:57.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:57 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:57.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:57 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.028s CPU time. 2024-04-05T02:55:57.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:57 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:55:57.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 32 16 64111 64095 125.169 76.3125 0.00573263 0.00798362 2024-04-05T02:55:58.128 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:57 smithi196 podman[85244]: 2024-04-05 02:55:57.768332481 +0000 UTC m=+0.209161607 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:55:58.128 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:57 smithi196 podman[85244]: 2024-04-05 02:55:57.893289909 +0000 UTC m=+0.334119029 container create 8887f81f186eca0b816fd721247e3a7d0f0c0e1ed30fa8bf82fe69dcf7d269fa (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, io.openshift.expose-services=, maintainer=Paul Cuzner , release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8) 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: from='client.25225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: [05/Apr/2024:02:55:56] ENGINE Serving on http://172.21.15.150:8765 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: pgmap v3: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: [05/Apr/2024:02:55:56] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: [05/Apr/2024:02:55:56] ENGINE Bus STARTED 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T02:55:58.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-mon[33169]: Cluster is now healthy 2024-04-05T02:55:58.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:58 smithi196 podman[85244]: 2024-04-05 02:55:58.247212179 +0000 UTC m=+0.688041298 container init 8887f81f186eca0b816fd721247e3a7d0f0c0e1ed30fa8bf82fe69dcf7d269fa (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., maintainer=Paul Cuzner , release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, distribution-scope=public, architecture=x86_64) 2024-04-05T02:55:58.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:58 smithi196 podman[85244]: 2024-04-05 02:55:58.255526807 +0000 UTC m=+0.696355926 container start 8887f81f186eca0b816fd721247e3a7d0f0c0e1ed30fa8bf82fe69dcf7d269fa (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, architecture=x86_64) 2024-04-05T02:55:58.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:58 smithi196 bash[85244]: 8887f81f186eca0b816fd721247e3a7d0f0c0e1ed30fa8bf82fe69dcf7d269fa 2024-04-05T02:55:58.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:58 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:55:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: from='client.25225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:55:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: [05/Apr/2024:02:55:56] ENGINE Serving on http://172.21.15.150:8765 2024-04-05T02:55:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: pgmap v3: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: [05/Apr/2024:02:55:56] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: [05/Apr/2024:02:55:56] ENGINE Bus STARTED 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[103445]: Cluster is now healthy 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: from='client.25225 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: [05/Apr/2024:02:55:56] ENGINE Serving on http://172.21.15.150:8765 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: pgmap v3: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: [05/Apr/2024:02:55:56] ENGINE Serving on https://172.21.15.150:7150 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: [05/Apr/2024:02:55:56] ENGINE Bus STARTED 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-04-05T02:55:58.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:58 smithi150 ceph-mon[41818]: Cluster is now healthy 2024-04-05T02:55:58.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 33 16 65404 65388 123.825 80.8125 0.0054643 0.00806213 2024-04-05T02:55:58.886 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[85271]: logger=settings t=2024-04-05T02:55:58.501791092Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:55:59.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:59 smithi196 ceph-mon[33169]: pgmap v4: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:55:59.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:55:59 smithi196 ceph-mon[33169]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-05T02:55:59.388 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:55:59 smithi196 podman[85330]: 2024-04-05 02:55:59.275660559 +0000 UTC m=+0.763147820 container died 8887f81f186eca0b816fd721247e3a7d0f0c0e1ed30fa8bf82fe69dcf7d269fa (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, distribution-scope=public, name=ubi8, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vendor=Red Hat, Inc., version=8.5, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, io.buildah.version=1.32.0, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , io.openshift.tags=base rhel8) 2024-04-05T02:55:59.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:59 smithi150 ceph-mon[41818]: pgmap v4: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:55:59.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:55:59 smithi150 ceph-mon[41818]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-05T02:55:59.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:59 smithi150 ceph-mon[103445]: pgmap v4: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:55:59.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:55:59 smithi150 ceph-mon[103445]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-05T02:55:59.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 34 16 66570 66554 122.326 72.875 0.00615324 0.00817021 2024-04-05T02:56:00.390 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:00 smithi196 podman[85330]: 2024-04-05 02:56:00.112921757 +0000 UTC m=+1.600409008 container remove 8887f81f186eca0b816fd721247e3a7d0f0c0e1ed30fa8bf82fe69dcf7d269fa (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Paul Cuzner , architecture=x86_64, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, com.redhat.component=ubi8-container, description=Ceph Grafana Container, version=8.5, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-05T02:56:00.390 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:00 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:56:00.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 35 16 67917 67901 121.236 84.1875 0.0220195 0.00824276 2024-04-05T02:56:00.935 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T02:56:01.135 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:00 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:56:01.136 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:00 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.055s CPU time. 2024-04-05T02:56:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:01 smithi150 ceph-mon[103445]: pgmap v5: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:56:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:01 smithi150 ceph-mon[103445]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-05T02:56:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:01 smithi150 ceph-mon[41818]: pgmap v5: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:56:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:01 smithi150 ceph-mon[41818]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-05T02:56:01.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:01 smithi196 ceph-mon[33169]: pgmap v5: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail 2024-04-05T02:56:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:01 smithi196 ceph-mon[33169]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-05T02:56:01.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 36 16 69088 69072 119.902 73.1875 0.0153087 0.00833434 2024-04-05T02:56:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:56:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:56:01] "GET /metrics HTTP/1.1" 200 33642 "" "Prometheus/2.43.0" 2024-04-05T02:56:02.015 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:56:02.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 37 16 70363 70347 118.815 79.6875 0.0129444 0.00841068 2024-04-05T02:56:03.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:03 smithi150 ceph-mon[41818]: pgmap v6: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-05T02:56:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:03 smithi150 ceph-mon[103445]: pgmap v6: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-05T02:56:03.579 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 38 16 71729 71713 117.934 85.375 0.00417147 0.00847449 2024-04-05T02:56:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:03 smithi196 ceph-mon[33169]: pgmap v6: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail 2024-04-05T02:56:04.580 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 39 16 72994 72978 116.937 79.0625 0.0117157 0.00854554 2024-04-05T02:56:04.734 INFO:teuthology.orchestra.run.smithi150.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:56:05.580 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:56:05.581112+0000 min lat: 0.00125779 max lat: 0.137698 avg lat: 0.00861114 2024-04-05T02:56:05.580 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:56:05.580 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 40 16 74300 74284 116.054 81.625 0.0100747 0.00861114 2024-04-05T02:56:05.698 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'sleep 60' 2024-04-05T02:56:05.698 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[103445]: pgmap v7: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 30 KiB/s rd, 56 MiB/s wr, 908 op/s 2024-04-05T02:56:05.698 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.698 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.698 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.699 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[41818]: pgmap v7: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 30 KiB/s rd, 56 MiB/s wr, 908 op/s 2024-04-05T02:56:05.699 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.699 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.699 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:05 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:05 smithi196 ceph-mon[33169]: pgmap v7: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 30 KiB/s rd, 56 MiB/s wr, 908 op/s 2024-04-05T02:56:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:05 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:05 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:05 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:06.580 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 41 16 75415 75399 114.923 69.6875 0.00385955 0.00869431 2024-04-05T02:56:06.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[103445]: from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:06.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[103445]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:56:06.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:06.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[41818]: from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[41818]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:56:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:06.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:06 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:06.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:06 smithi196 ceph-mon[33169]: from='client.25325 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:56:06.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:06 smithi196 ceph-mon[33169]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:56:06.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:06 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:06.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:06 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:07.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 42 16 76743 76727 114.158 83 0.0131483 0.00875458 2024-04-05T02:56:07.627 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:07 smithi150 ceph-mon[103445]: pgmap v8: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 KiB/s rd, 43 MiB/s wr, 706 op/s 2024-04-05T02:56:07.627 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:07 smithi150 ceph-mon[41818]: pgmap v8: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 KiB/s rd, 43 MiB/s wr, 706 op/s 2024-04-05T02:56:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:07 smithi196 ceph-mon[33169]: pgmap v8: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 23 KiB/s rd, 43 MiB/s wr, 706 op/s 2024-04-05T02:56:07.874 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:56:08.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 43 16 77940 77924 113.243 74.8125 0.0441062 0.00882414 2024-04-05T02:56:08.883 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:56:08 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:56:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:56:09.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 44 16 79073 79057 112.279 70.8125 0.0169263 0.00890075 2024-04-05T02:56:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:09 smithi150 ceph-mon[103445]: pgmap v9: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 71 MiB/s wr, 1.15k op/s 2024-04-05T02:56:09.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:09 smithi150 ceph-mon[41818]: pgmap v9: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 71 MiB/s wr, 1.15k op/s 2024-04-05T02:56:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:09 smithi196 ceph-mon[33169]: pgmap v9: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 19 KiB/s rd, 71 MiB/s wr, 1.15k op/s 2024-04-05T02:56:10.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 45 16 80143 80127 111.27 66.875 0.0125834 0.00898169 2024-04-05T02:56:11.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:11 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-04-05T02:56:11.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:11 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:11.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:11 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.055s CPU time. 2024-04-05T02:56:11.387 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:11 smithi196 systemd[1]: Starting Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:56:11.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: pgmap v10: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 65 MiB/s wr, 1.05k op/s 2024-04-05T02:56:11.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:11.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: pgmap v10: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 65 MiB/s wr, 1.05k op/s 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: pgmap v10: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 65 MiB/s wr, 1.05k op/s 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:11.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:11.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 46 16 81345 81329 110.484 75.125 0.00740078 0.00904655 2024-04-05T02:56:11.816 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:11 smithi196 podman[88492]: 2024-04-05 02:56:11.366692901 +0000 UTC m=+0.032900069 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-04-05T02:56:11.816 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:11 smithi196 podman[88492]: 2024-04-05 02:56:11.483394625 +0000 UTC m=+0.149601801 container create 5941f25e1de05ed87e422d081b19a082ada8f253af17ecae8a432798760dfb65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , version=8.5, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.expose-services=, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base rhel8, architecture=x86_64) 2024-04-05T02:56:11.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:56:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:56:11] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-05T02:56:12.136 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 podman[88492]: 2024-04-05 02:56:12.028306863 +0000 UTC m=+0.694514037 container init 5941f25e1de05ed87e422d081b19a082ada8f253af17ecae8a432798760dfb65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, architecture=x86_64, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8.5) 2024-04-05T02:56:12.137 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 podman[88492]: 2024-04-05 02:56:12.035707546 +0000 UTC m=+0.701914720 container start 5941f25e1de05ed87e422d081b19a082ada8f253af17ecae8a432798760dfb65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, vcs-type=git, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, maintainer=Paul Cuzner , description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, release=239.1651231664) 2024-04-05T02:56:12.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 47 16 82472 82456 109.632 70.4375 0.00941371 0.00911606 2024-04-05T02:56:12.611 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 bash[88492]: 5941f25e1de05ed87e422d081b19a082ada8f253af17ecae8a432798760dfb65 2024-04-05T02:56:12.611 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 systemd[1]: Started Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:12.611 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a[88506]: logger=settings t=2024-04-05T02:56:12.26117836Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-04-05T02:56:12.611 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 podman[88543]: 2024-04-05 02:56:12.307008879 +0000 UTC m=+0.034004487 container died 5941f25e1de05ed87e422d081b19a082ada8f253af17ecae8a432798760dfb65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., version=8.5, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.openshift.tags=base rhel8, vcs-type=git, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-04-05T02:56:13.277 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:13 smithi196 ceph-mon[33169]: pgmap v11: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 65 MiB/s wr, 1.05k op/s 2024-04-05T02:56:13.277 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:13 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:13.277 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:13 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:13.277 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:13 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:13.277 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:13 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:56:13.278 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 podman[88543]: 2024-04-05 02:56:12.910858386 +0000 UTC m=+0.637853982 container remove 5941f25e1de05ed87e422d081b19a082ada8f253af17ecae8a432798760dfb65 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, description=Ceph Grafana Container, name=ubi8, com.redhat.component=ubi8-container, io.openshift.expose-services=, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.buildah.version=1.32.0, distribution-scope=public, vendor=Red Hat, Inc., maintainer=Paul Cuzner , version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-04-05T02:56:13.278 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:12 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-04-05T02:56:13.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[103445]: pgmap v11: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 65 MiB/s wr, 1.05k op/s 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[41818]: pgmap v11: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 17 KiB/s rd, 65 MiB/s wr, 1.05k op/s 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:13.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:13 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:56:13.582 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 48 16 83586 83570 108.798 69.625 0.0161163 0.00918568 2024-04-05T02:56:13.636 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:13 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:56:13.637 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:13 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.014s CPU time. 2024-04-05T02:56:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:14 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:56:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:14 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:56:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:14 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:56:14.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:14 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[103445]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[103445]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[103445]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[103445]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:56:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:14 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.conf 2024-04-05T02:56:14.583 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 49 16 84672 84656 107.96 67.875 0.00965495 0.0092574 2024-04-05T02:56:15.583 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 50 16 85895 85879 107.329 76.4375 0.00658141 0.00931196 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: pgmap v12: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 96 MiB/s wr, 1.55k op/s 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: pgmap v12: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 96 MiB/s wr, 1.55k op/s 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:15.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:15.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:56:15.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:15 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: pgmap v12: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 17 KiB/s rd, 96 MiB/s wr, 1.55k op/s 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: Updating smithi196:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: Updating smithi150:/etc/ceph/ceph.client.admin.keyring 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: Updating smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: Updating smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:15.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:56:15.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:56:15.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:56:15.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:15 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:16.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 51 16 87153 87137 106.766 78.625 0.0124875 0.00936174 2024-04-05T02:56:16.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[103445]: pgmap v13: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[103445]: pgmap v14: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[103445]: pgmap v15: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[103445]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[41818]: pgmap v13: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[41818]: pgmap v14: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[41818]: pgmap v15: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-05T02:56:16.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:16 smithi150 ceph-mon[41818]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T02:56:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:16 smithi196 ceph-mon[33169]: pgmap v13: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-04-05T02:56:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:16 smithi196 ceph-mon[33169]: pgmap v14: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-05T02:56:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:16 smithi196 ceph-mon[33169]: pgmap v15: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 807 op/s 2024-04-05T02:56:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:16 smithi196 ceph-mon[33169]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-04-05T02:56:17.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 52 16 88348 88332 106.15 74.6875 0.00825157 0.00941507 2024-04-05T02:56:18.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 53 16 89593 89577 105.615 77.8125 0.0159948 0.00946345 2024-04-05T02:56:18.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:18 smithi150 ceph-mon[41818]: pgmap v16: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-05T02:56:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:18 smithi150 ceph-mon[103445]: pgmap v16: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-05T02:56:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:18 smithi196 ceph-mon[33169]: pgmap v16: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-05T02:56:18.887 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:56:18 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:56:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:56:19.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 54 16 90789 90773 105.043 74.75 0.0195192 0.00951483 2024-04-05T02:56:20.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 55 16 92005 91989 104.515 76 0.0109759 0.00956282 2024-04-05T02:56:20.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:20 smithi150 ceph-mon[103445]: pgmap v17: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-05T02:56:20.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:20 smithi150 ceph-mon[41818]: pgmap v17: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-05T02:56:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:20 smithi196 ceph-mon[33169]: pgmap v17: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s 2024-04-05T02:56:21.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 56 16 93206 93190 103.989 75.0625 0.0116573 0.0096112 2024-04-05T02:56:21.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:56:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:56:21] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2024-04-05T02:56:22.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 57 16 94396 94380 103.469 74.375 0.02 0.00965946 2024-04-05T02:56:22.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:22 smithi150 ceph-mon[103445]: pgmap v18: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-04-05T02:56:22.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:22 smithi150 ceph-mon[41818]: pgmap v18: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-04-05T02:56:22.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:22 smithi196 ceph-mon[33169]: pgmap v18: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s 2024-04-05T02:56:23.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 58 16 95536 95520 102.914 71.25 0.01133 0.0097121 2024-04-05T02:56:23.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:23 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-04-05T02:56:23.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:23 smithi196 systemd[1]: Stopped Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:23.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:23 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Consumed 1.014s CPU time. 2024-04-05T02:56:23.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:23 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Start request repeated too quickly. 2024-04-05T02:56:23.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:23 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@grafana.a.service: Failed with result 'exit-code'. 2024-04-05T02:56:23.887 INFO:journalctl@ceph.grafana.a.smithi196.stdout:Apr 05 02:56:23 smithi196 systemd[1]: Failed to start Ceph grafana.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:56:24.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 59 16 96717 96701 102.42 73.8125 0.00963256 0.00975835 2024-04-05T02:56:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:24 smithi196 ceph-mon[33169]: pgmap v19: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T02:56:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:24 smithi150 ceph-mon[103445]: pgmap v19: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T02:56:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:24 smithi150 ceph-mon[41818]: pgmap v19: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T02:56:25.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:56:25.585888+0000 min lat: 0.00125779 max lat: 0.137698 avg lat: 0.00980744 2024-04-05T02:56:25.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:56:25.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 60 16 97867 97851 101.911 71.875 0.0146464 0.00980744 2024-04-05T02:56:25.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:25 smithi196 ceph-mon[33169]: pgmap v20: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-05T02:56:25.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:25 smithi150 ceph-mon[41818]: pgmap v20: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-05T02:56:25.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:25 smithi150 ceph-mon[103445]: pgmap v20: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s 2024-04-05T02:56:26.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 61 16 99013 98997 101.415 71.625 0.0173393 0.00985549 2024-04-05T02:56:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:26.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:26 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:26.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:26 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:27.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:26 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:56:27.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:26 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:27.584 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 62 16 100139 100123 100.914 70.375 0.00602101 0.00990463 2024-04-05T02:56:27.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:27 smithi150 ceph-mon[103445]: pgmap v21: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-05T02:56:27.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:27 smithi150 ceph-mon[41818]: pgmap v21: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-05T02:56:28.037 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:27 smithi196 ceph-mon[33169]: pgmap v21: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-05T02:56:28.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 63 16 101239 101223 100.403 68.75 0.00899995 0.00995391 2024-04-05T02:56:29.048 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:56:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:56:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:56:29.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 64 16 102346 102330 99.9156 69.1875 0.0166167 0.0100035 2024-04-05T02:56:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:29 smithi150 ceph-mon[103445]: pgmap v22: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-04-05T02:56:30.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:29 smithi150 ceph-mon[41818]: pgmap v22: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-04-05T02:56:30.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:29 smithi196 ceph-mon[33169]: pgmap v22: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-04-05T02:56:30.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 65 16 103435 103419 99.4255 68.0625 0.00505705 0.0100526 2024-04-05T02:56:31.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 66 16 104525 104509 98.9512 68.125 0.0238219 0.0101009 2024-04-05T02:56:31.931 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:56:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:56:31] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-05T02:56:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:31 smithi150 ceph-mon[41818]: pgmap v23: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-04-05T02:56:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:31 smithi150 ceph-mon[103445]: pgmap v23: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-04-05T02:56:32.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:31 smithi196 ceph-mon[33169]: pgmap v23: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-04-05T02:56:32.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 67 16 105597 105581 98.4742 67 0.00850171 0.0101493 2024-04-05T02:56:33.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 68 16 106675 106659 98.0168 67.375 0.0149162 0.0101973 2024-04-05T02:56:34.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:33 smithi150 ceph-mon[41818]: pgmap v24: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T02:56:34.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:33 smithi150 ceph-mon[103445]: pgmap v24: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T02:56:34.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:33 smithi196 ceph-mon[33169]: pgmap v24: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T02:56:34.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 69 16 107743 107727 97.5637 66.75 0.00519398 0.0102445 2024-04-05T02:56:35.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 70 16 108808 108792 97.1207 66.5625 0.01185 0.0102915 2024-04-05T02:56:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:35 smithi150 ceph-mon[103445]: pgmap v25: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2024-04-05T02:56:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:35 smithi150 ceph-mon[41818]: pgmap v25: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2024-04-05T02:56:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:35 smithi196 ceph-mon[33169]: pgmap v25: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2024-04-05T02:56:36.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 71 16 109851 109835 96.6709 65.1875 0.0149538 0.0103395 2024-04-05T02:56:37.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 72 16 110899 110883 96.2379 65.5 0.00748273 0.0103852 2024-04-05T02:56:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:37 smithi150 ceph-mon[103445]: pgmap v26: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-05T02:56:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:37 smithi150 ceph-mon[41818]: pgmap v26: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-05T02:56:38.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:37 smithi196 ceph-mon[33169]: pgmap v26: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-04-05T02:56:38.585 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 73 16 111936 111920 95.8074 64.8125 0.00945693 0.0104327 2024-04-05T02:56:39.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:56:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:56:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:56:39.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 74 16 112899 112883 95.326 60.1875 0.00623885 0.0104848 2024-04-05T02:56:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:39 smithi150 ceph-mon[103445]: pgmap v27: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-04-05T02:56:39.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:39 smithi150 ceph-mon[41818]: pgmap v27: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-04-05T02:56:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:39 smithi196 ceph-mon[33169]: pgmap v27: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-04-05T02:56:40.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 75 16 113790 113774 94.7975 55.6875 0.00813483 0.0105434 2024-04-05T02:56:41.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:41.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:40 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:40 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:41.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 76 16 114736 114720 94.3281 59.125 0.010074 0.0105964 2024-04-05T02:56:41.973 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:56:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:56:41] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-05T02:56:42.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:41 smithi150 ceph-mon[41818]: pgmap v28: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-04-05T02:56:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:41 smithi150 ceph-mon[103445]: pgmap v28: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-04-05T02:56:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:41 smithi196 ceph-mon[33169]: pgmap v28: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s 2024-04-05T02:56:42.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 77 16 115744 115728 93.9212 63 0.0222208 0.0106417 2024-04-05T02:56:43.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 78 16 116741 116725 93.5159 62.3125 0.0205041 0.0106883 2024-04-05T02:56:44.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:43 smithi150 ceph-mon[103445]: pgmap v29: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-05T02:56:44.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:43 smithi150 ceph-mon[41818]: pgmap v29: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-05T02:56:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:43 smithi196 ceph-mon[33169]: pgmap v29: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-04-05T02:56:44.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 79 16 117729 117713 93.1137 61.75 0.00333717 0.010732 2024-04-05T02:56:45.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T02:56:45.587650+0000 min lat: 0.00125779 max lat: 0.137698 avg lat: 0.0107818 2024-04-05T02:56:45.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:56:45.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 80 16 118696 118680 92.7052 60.4375 0.0191156 0.0107818 2024-04-05T02:56:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:45 smithi196 ceph-mon[33169]: pgmap v30: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-04-05T02:56:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:45 smithi150 ceph-mon[41818]: pgmap v30: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-04-05T02:56:46.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:45 smithi150 ceph-mon[103445]: pgmap v30: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s 2024-04-05T02:56:46.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 81 16 119673 119657 92.3145 61.0625 0.00944891 0.0108273 2024-04-05T02:56:47.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 82 16 120664 120648 91.944 61.9375 0.00726721 0.010871 2024-04-05T02:56:48.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:47 smithi196 ceph-mon[33169]: pgmap v31: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-05T02:56:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:47 smithi150 ceph-mon[41818]: pgmap v31: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-05T02:56:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:47 smithi150 ceph-mon[103445]: pgmap v31: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-04-05T02:56:48.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 83 16 121652 121636 91.5802 61.75 0.0133854 0.0109141 2024-04-05T02:56:49.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:56:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:56:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:56:49.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 84 16 122626 122610 91.2146 60.875 0.0570984 0.0109572 2024-04-05T02:56:50.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:49 smithi196 ceph-mon[33169]: pgmap v32: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-05T02:56:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:49 smithi150 ceph-mon[41818]: pgmap v32: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-05T02:56:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:49 smithi150 ceph-mon[103445]: pgmap v32: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-05T02:56:50.586 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 85 16 123580 123564 90.8429 59.625 0.00936867 0.0110033 2024-04-05T02:56:51.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 86 16 124510 124494 90.4624 58.125 0.009958 0.0110491 2024-04-05T02:56:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:56:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:56:51] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-05T02:56:52.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:51 smithi196 ceph-mon[33169]: pgmap v33: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-05T02:56:52.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:51 smithi150 ceph-mon[41818]: pgmap v33: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-05T02:56:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:51 smithi150 ceph-mon[103445]: pgmap v33: 89 pgs: 89 active+clean; 7.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-04-05T02:56:52.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 87 16 125444 125428 90.0936 58.375 0.0246063 0.0110943 2024-04-05T02:56:53.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 88 16 126397 126381 89.7466 59.5625 0.0171503 0.0111376 2024-04-05T02:56:54.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:53 smithi196 ceph-mon[33169]: pgmap v34: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-05T02:56:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:53 smithi150 ceph-mon[41818]: pgmap v34: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-05T02:56:54.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:53 smithi150 ceph-mon[103445]: pgmap v34: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-04-05T02:56:54.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 89 16 127347 127331 89.4053 59.375 0.00823286 0.01118 2024-04-05T02:56:55.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total time run: 90.0111 2024-04-05T02:56:55.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total writes made: 128281 2024-04-05T02:56:55.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Write size: 65536 2024-04-05T02:56:55.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object size: 65536 2024-04-05T02:56:55.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Bandwidth (MB/sec): 89.0731 2024-04-05T02:56:55.587 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Bandwidth: 38.4685 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max bandwidth (MB/sec): 243.375 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min bandwidth (MB/sec): 55.6875 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average IOPS: 1425 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev IOPS: 615.495 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max IOPS: 3894 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min IOPS: 891 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average Latency(s): 0.0112224 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Latency(s): 0.0073773 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max latency(s): 0.137698 2024-04-05T02:56:55.588 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min latency(s): 0.00125779 2024-04-05T02:56:55.593 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Cleaning up (deleting benchmark objects) 2024-04-05T02:56:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:56 smithi196 ceph-mon[33169]: pgmap v35: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2024-04-05T02:56:56.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:56 smithi150 ceph-mon[41818]: pgmap v35: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2024-04-05T02:56:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:56 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:56.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:56 smithi150 ceph-mon[103445]: pgmap v35: 89 pgs: 89 active+clean; 7.6 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 815 op/s 2024-04-05T02:56:56.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:56:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:56:58 smithi196 ceph-mon[33169]: pgmap v36: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 71 MiB/s wr, 1.21k op/s 2024-04-05T02:56:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:56:58 smithi150 ceph-mon[41818]: pgmap v36: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 71 MiB/s wr, 1.21k op/s 2024-04-05T02:56:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:56:58 smithi150 ceph-mon[103445]: pgmap v36: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 71 MiB/s wr, 1.21k op/s 2024-04-05T02:56:59.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:56:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:56:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:00 smithi196 ceph-mon[33169]: pgmap v37: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 799 op/s 2024-04-05T02:57:00.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:00 smithi150 ceph-mon[103445]: pgmap v37: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 799 op/s 2024-04-05T02:57:00.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:00 smithi150 ceph-mon[41818]: pgmap v37: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 799 op/s 2024-04-05T02:57:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:57:01] "GET /metrics HTTP/1.1" 200 36359 "" "Prometheus/2.43.0" 2024-04-05T02:57:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:02 smithi196 ceph-mon[33169]: pgmap v38: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 799 op/s 2024-04-05T02:57:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:02 smithi150 ceph-mon[103445]: pgmap v38: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 799 op/s 2024-04-05T02:57:02.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:02 smithi150 ceph-mon[41818]: pgmap v38: 89 pgs: 89 active+clean; 7.8 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 799 op/s 2024-04-05T02:57:04.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:04 smithi196 ceph-mon[33169]: pgmap v39: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.20k op/s 2024-04-05T02:57:04.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:04 smithi150 ceph-mon[41818]: pgmap v39: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.20k op/s 2024-04-05T02:57:04.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:04 smithi150 ceph-mon[103445]: pgmap v39: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 1.20k op/s 2024-04-05T02:57:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:06 smithi196 ceph-mon[33169]: pgmap v40: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 797 op/s 2024-04-05T02:57:06.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:06 smithi150 ceph-mon[103445]: pgmap v40: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 797 op/s 2024-04-05T02:57:06.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:06 smithi150 ceph-mon[41818]: pgmap v40: 89 pgs: 89 active+clean; 7.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 797 op/s 2024-04-05T02:57:08.279 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:08 smithi150 ceph-mon[103445]: pgmap v41: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 1.20k op/s 2024-04-05T02:57:08.279 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:08 smithi150 ceph-mon[41818]: pgmap v41: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 1.20k op/s 2024-04-05T02:57:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:08 smithi196 ceph-mon[33169]: pgmap v41: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 20 MiB/s wr, 1.20k op/s 2024-04-05T02:57:09.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:57:08 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:57:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:10.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:10 smithi196 ceph-mon[33169]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-04-05T02:57:10.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:10 smithi150 ceph-mon[103445]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-04-05T02:57:10.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:10 smithi150 ceph-mon[41818]: pgmap v42: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-04-05T02:57:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:11 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:11.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:11.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:11 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:11.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:57:11] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-05T02:57:12.360 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:12 smithi150 ceph-mon[41818]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-04-05T02:57:12.360 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:12 smithi150 ceph-mon[103445]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-04-05T02:57:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:12 smithi196 ceph-mon[33169]: pgmap v43: 89 pgs: 89 active+clean; 7.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2024-04-05T02:57:13.335 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade resume' 2024-04-05T02:57:13.807 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:14 smithi150 ceph-mon[41818]: pgmap v44: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-05T02:57:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:14 smithi150 ceph-mon[103445]: pgmap v44: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-05T02:57:14.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:14 smithi196 ceph-mon[33169]: pgmap v44: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s 2024-04-05T02:57:15.379 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:15.379 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:15 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:15.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:15 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:16 smithi196 ceph-mon[33169]: pgmap v45: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-04-05T02:57:16.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:16 smithi150 ceph-mon[41818]: pgmap v45: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-04-05T02:57:16.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:16 smithi150 ceph-mon[103445]: pgmap v45: 89 pgs: 89 active+clean; 6.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-04-05T02:57:17.273 INFO:teuthology.orchestra.run.smithi150.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:18.152 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-05T02:57:18.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:18 smithi196 ceph-mon[33169]: pgmap v46: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-05T02:57:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:18 smithi196 ceph-mon[33169]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:18 smithi196 ceph-mon[33169]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:18 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[103445]: pgmap v46: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[103445]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[103445]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[41818]: pgmap v46: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[41818]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[41818]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:18 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:18.627 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:19.133 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:57:18 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:57:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:20.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:20 smithi196 ceph-mon[33169]: pgmap v47: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-05T02:57:20.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:20 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:20.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:20 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:20.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:20 smithi150 ceph-mon[103445]: pgmap v47: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-05T02:57:20.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:20 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:20.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:20 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:20.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:20 smithi150 ceph-mon[41818]: pgmap v47: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-05T02:57:20.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:20 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:20.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:20 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:57:21] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2024-04-05T02:57:21.850 INFO:teuthology.orchestra.run.smithi150.stdout:wait for majority of mons to upgrade 2024-04-05T02:57:22.555 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:22 smithi150 ceph-mon[41818]: pgmap v48: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-05T02:57:22.556 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:22 smithi150 ceph-mon[103445]: pgmap v48: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-05T02:57:22.597 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-05T02:57:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:22 smithi196 ceph-mon[33169]: pgmap v48: 89 pgs: 89 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 730 op/s 2024-04-05T02:57:23.456 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:24.352 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:24 smithi150 ceph-mon[41818]: pgmap v49: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:57:24.352 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:24 smithi150 ceph-mon[103445]: pgmap v49: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:57:24.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:24 smithi196 ceph-mon[33169]: pgmap v49: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-04-05T02:57:26.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[103445]: pgmap v50: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[41818]: pgmap v50: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:26.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:26 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:26 smithi196 ceph-mon[33169]: pgmap v50: 89 pgs: 89 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 689 op/s 2024-04-05T02:57:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:26 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:26 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:26 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/22854726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:57:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:27 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-05T02:57:27.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:27.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/22854726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/22854726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:27.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:27.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:57:27.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:57:27.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:27 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-05T02:57:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:28 smithi150 ceph-mon[103445]: pgmap v51: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 964 op/s 2024-04-05T02:57:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:28 smithi150 ceph-mon[103445]: Upgrade: It appears safe to stop mon.c 2024-04-05T02:57:28.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:28 smithi150 ceph-mon[41818]: pgmap v51: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 964 op/s 2024-04-05T02:57:28.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:28 smithi150 ceph-mon[41818]: Upgrade: It appears safe to stop mon.c 2024-04-05T02:57:28.539 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:28 smithi196 ceph-mon[33169]: pgmap v51: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 964 op/s 2024-04-05T02:57:28.539 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:28 smithi196 ceph-mon[33169]: Upgrade: It appears safe to stop mon.c 2024-04-05T02:57:28.886 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:57:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:57:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:29.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:29 smithi196 ceph-mon[33169]: mgrmap e40: y(active, since 92s), standbys: x 2024-04-05T02:57:29.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:29 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:29.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:29 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:29.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:29 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:29.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:29 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[41818]: mgrmap e40: y(active, since 92s), standbys: x 2024-04-05T02:57:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[41818]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[41818]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[103445]: mgrmap e40: y(active, since 92s), standbys: x 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:57:29.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:29 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:30.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:30 smithi196 ceph-mon[33169]: pgmap v52: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 696 op/s 2024-04-05T02:57:30.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:30 smithi196 ceph-mon[33169]: Upgrade: Updating mon.c 2024-04-05T02:57:30.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:30 smithi196 ceph-mon[33169]: Deploying daemon mon.c on smithi150 2024-04-05T02:57:30.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:30 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3127387325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:30.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[41818]: pgmap v52: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 696 op/s 2024-04-05T02:57:30.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[41818]: Upgrade: Updating mon.c 2024-04-05T02:57:30.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[41818]: Deploying daemon mon.c on smithi150 2024-04-05T02:57:30.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[41818]: from='client.? 172.21.15.150:0/3127387325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[103445]: pgmap v52: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 696 op/s 2024-04-05T02:57:30.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[103445]: Upgrade: Updating mon.c 2024-04-05T02:57:30.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[103445]: Deploying daemon mon.c on smithi150 2024-04-05T02:57:30.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:30 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3127387325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:31.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:31 smithi150 systemd[1]: Stopping Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:57:31.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:57:31] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-05T02:57:32.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c[41795]: 2024-04-05T02:57:32.146+0000 7f2ff9559700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T02:57:32.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c[41795]: 2024-04-05T02:57:32.146+0000 7f2ff9559700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:32 smithi150 podman[107509]: 2024-04-05 02:57:32.369312718 +0000 UTC m=+0.368073326 container died 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, RELEASE=quincy-6fda41f, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS) 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:57:32] ENGINE Bus STOPPING 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:57:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:57:32] ENGINE Bus STOPPED 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:57:32] ENGINE Bus STARTING 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:57:32] ENGINE Serving on http://:::9283 2024-04-05T02:57:32.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:57:32] ENGINE Bus STARTED 2024-04-05T02:57:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:32 smithi150 podman[107509]: 2024-04-05 02:57:32.809039019 +0000 UTC m=+0.807799617 container cleanup 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-05T02:57:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:32 smithi150 bash[107509]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c 2024-04-05T02:57:33.441 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:33 smithi150 podman[107538]: 2024-04-05 02:57:33.044734452 +0000 UTC m=+0.679628117 container remove 826b3558df5dbcd2634b17804cb1441a7102245553448f63fbff26b70ae6a3a3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, RELEASE=quincy-6fda41f, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD) 2024-04-05T02:57:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:33 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.c.service: Deactivated successfully. 2024-04-05T02:57:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:33 smithi150 systemd[1]: Stopped Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:57:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:33 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.c.service: Consumed 9.877s CPU time. 2024-04-05T02:57:34.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:33 smithi150 systemd[1]: Starting Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:57:34.458 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 podman[107651]: 2024-04-05 02:57:34.058453067 +0000 UTC m=+0.057552889 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:34.458 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 podman[107651]: 2024-04-05 02:57:34.179418074 +0000 UTC m=+0.178517866 container create 74316b4d98ae1d1cfbefb4d99a35e704a5027422cb578c2458d77437c175bc76 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-05T02:57:34.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 podman[107651]: 2024-04-05 02:57:34.458894445 +0000 UTC m=+0.457994235 container init 74316b4d98ae1d1cfbefb4d99a35e704a5027422cb578c2458d77437c175bc76 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-04-05T02:57:34.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 podman[107651]: 2024-04-05 02:57:34.470080612 +0000 UTC m=+0.469180401 container start 74316b4d98ae1d1cfbefb4d99a35e704a5027422cb578c2458d77437c175bc76 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T02:57:34.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:57:34.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev), process ceph-mon, pid 7 2024-04-05T02:57:34.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: pidfile_write: ignore empty --pid-file 2024-04-05T02:57:34.733 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: load: jerasure load: lrc 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: RocksDB version: 7.9.2 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Git sha 0 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Compile date 2024-04-04 02:01:22 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: DB SUMMARY 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: DB Session ID: FGNI49XS4MYGLO3S0SD0 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1092 Bytes 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 7712869 ; 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:57:34.734 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.env: 0x55f81e765c60 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.fs: PosixFileSystem 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.info_log: 0x55f81f477ae0 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.statistics: (nil) 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.use_fsync: 0 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:57:34.735 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.db_log_dir: 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.wal_dir: 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.write_buffer_manager: 0x55f81f44a820 2024-04-05T02:57:34.736 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.unordered_write: 0 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.row_cache: None 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.wal_filter: None 2024-04-05T02:57:34.737 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.wal_compression: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:57:34.738 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_open_files: -1 2024-04-05T02:57:34.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Compression algorithms supported: 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kZSTD supported: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-05T02:57:34.740 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.merge_operator: 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_filter: None 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f81f477be0) 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: index_type: 0 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: data_block_index_type: 0 2024-04-05T02:57:34.741 INFO:journalctl@ceph.mon.c.smithi150.stdout: index_shortening: 1 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: checksum: 4 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: no_block_cache: 0 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache: 0x55f81f47b1f0 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache_options: 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: capacity : 536870912 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: num_shard_bits : 4 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: strict_capacity_limit : 0 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_cache_compressed: (nil) 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: persistent_cache: (nil) 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_size: 4096 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_size_deviation: 10 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_restart_interval: 16 2024-04-05T02:57:34.742 INFO:journalctl@ceph.mon.c.smithi150.stdout: index_block_restart_interval: 1 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: metadata_block_size: 4096 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: partition_filters: 0 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: use_delta_encoding: 1 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: filter_policy: bloomfilter 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: whole_key_filtering: 1 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: verify_compression: 0 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: format_version: 5 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: enable_index_compression: 1 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: block_align: 0 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: max_auto_readahead_size: 262144 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: prepopulate_block_cache: 0 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: initial_auto_readahead_size: 8192 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout: num_file_reads_for_auto_readahead: 2 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:57:34.743 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression: NoCompression 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.num_levels: 7 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:57:34.744 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:57:34.745 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.arena_block_size: 1048576 2024-04-05T02:57:34.746 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:57:34.747 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.ttl: 2592000 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:57:34.748 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.blob_file_starting_level: 0 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12681, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: bbac959f-e8fb-41cd-9fa9-6cae06e2a51d 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285854530138, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 bash[107651]: 74316b4d98ae1d1cfbefb4d99a35e704a5027422cb578c2458d77437c175bc76 2024-04-05T02:57:34.749 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285854560318, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 4740818, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12686, "largest_seqno": 15385, "table_properties": {"data_size": 4729671, "index_size": 7054, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3077, "raw_key_size": 25946, "raw_average_key_size": 21, "raw_value_size": 4705641, "raw_average_value_size": 3876, "num_data_blocks": 318, "num_entries": 1214, "num_filter_entries": 1214, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712285854, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "bbac959f-e8fb-41cd-9fa9-6cae06e2a51d", "db_session_id": "FGNI49XS4MYGLO3S0SD0", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285854560457, "job": 1, "event": "recovery_finished"} 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 systemd[1]: Started Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f81f568000 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: DB pointer 0x55f81f556000 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** DB Stats ** 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:57:34.750 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: L0 1/0 4.52 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 415.3 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: L6 1/0 6.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: Sum 2/0 10.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 415.3 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 415.3 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** Compaction Stats [default] ** 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 415.3 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-04-05T02:57:34.751 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Cumulative compaction: 0.00 GB write, 100.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Interval compaction: 0.00 GB write, 100.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Block cache BinnedLRUCache@0x55f81f47b1f0#7 capacity: 512.00 MB usage: 36.47 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,12.31 KB,0.00234842%) IndexBlock(2,24.16 KB,0.00460744%) Misc(2,6.58 KB,0.00125468%) 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:57:34.752 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: starting mon.c rank 2 at public addrs [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] at bind addrs [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???) e3 preinit fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).mds e9 new map 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).mds e9 print_map 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: e9 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: legacy client fscid: 1 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: Filesystem 'foo' (1) 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: fs_name foo 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: epoch 8 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: created 2024-04-05T02:49:47.190260+0000 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: modified 2024-04-05T02:50:03.911335+0000 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: tableserver 0 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: root 0 2024-04-05T02:57:34.753 INFO:journalctl@ceph.mon.c.smithi150.stdout: session_timeout 60 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: session_autoclose 300 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: max_file_size 1099511627776 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: max_xattr_size 65536 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: required_client_features {} 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: last_failure 0 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: last_failure_osd_epoch 0 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: max_mds 1 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: in 0 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: up {0=24557} 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: failed 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: damaged 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: stopped 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: data_pools [4] 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: metadata_pool 3 2024-04-05T02:57:34.754 INFO:journalctl@ceph.mon.c.smithi150.stdout: inline_data disabled 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: balancer 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: bal_rank_mask -1 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: standby_count_wanted 1 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: qdb_cluster leader: 0 members: 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: [mds.foo.smithi150.jnrwgs{0:24557} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: Standby daemons: 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout: [mds.foo.smithi196.eohomm{-1:24526} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:57:34.755 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:34 smithi150 ceph-mon[107701]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-05T02:57:39.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:57:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:57:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:39 smithi196 ceph-mon[33169]: pgmap v53: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 696 op/s 2024-04-05T02:57:39.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:39 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:39.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:39 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:39.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:39 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:39.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:39 smithi196 ceph-mon[33169]: from='client.25343 172.21.15.150:0/4059299798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:39 smithi150 ceph-mon[103445]: pgmap v53: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 696 op/s 2024-04-05T02:57:39.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:39 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:39.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:39 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:39.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:39 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:39.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:39 smithi150 ceph-mon[103445]: from='client.25343 172.21.15.150:0/4059299798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:40.824 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 2 2024-04-05T02:57:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:40 smithi150 ceph-mon[107701]: pgmap v53: 89 pgs: 89 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 260 B/s wr, 696 op/s 2024-04-05T02:57:40.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:57:40.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:57:40.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:57:40.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:40 smithi150 ceph-mon[107701]: from='client.25343 172.21.15.150:0/4059299798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: mon.c calling monitor election 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: mon.b calling monitor election 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: mon.a calling monitor election 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: monmap epoch 3 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: last_changed 2024-04-05T02:44:29.726178+0000 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: min_mon_release 17 (quincy) 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: election_strategy: 1 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:57:41.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: mgrmap e40: y(active, since 105s), standbys: x 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: daemon grafana.a on smithi196 is in error state 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: pgmap v58: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-04-05T02:57:41.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:41 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/2158264309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:57:41] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: mon.c calling monitor election 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: mon.b calling monitor election 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: mon.a calling monitor election 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: monmap epoch 3 2024-04-05T02:57:41.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: last_changed 2024-04-05T02:44:29.726178+0000 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: min_mon_release 17 (quincy) 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: election_strategy: 1 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: mgrmap e40: y(active, since 105s), standbys: x 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: daemon grafana.a on smithi196 is in error state 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: pgmap v58: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-04-05T02:57:41.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2158264309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: mon.c calling monitor election 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: mon.b calling monitor election 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: mon.a calling monitor election 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: monmap epoch 3 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: last_changed 2024-04-05T02:44:29.726178+0000 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: min_mon_release 17 (quincy) 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: election_strategy: 1 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:57:41.985 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: mgrmap e40: y(active, since 105s), standbys: x 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: daemon grafana.a on smithi196 is in error state 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: pgmap v58: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 673 op/s 2024-04-05T02:57:41.986 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2158264309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:57:42.911 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T02:57:43.891 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:43 smithi150 ceph-mon[107701]: pgmap v59: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:57:43.891 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:43 smithi150 ceph-mon[103445]: pgmap v59: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:57:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:43 smithi196 ceph-mon[33169]: pgmap v59: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:57:44.370 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:44.891 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:57:44.585+0000 7f45dfa59700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-05T02:57:45.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:45 smithi150 ceph-mon[107701]: pgmap v60: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-04-05T02:57:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:45 smithi150 ceph-mon[103445]: pgmap v60: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-04-05T02:57:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:45 smithi196 ceph-mon[33169]: pgmap v60: 89 pgs: 89 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2024-04-05T02:57:47.102 INFO:teuthology.orchestra.run.smithi150.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T02:57:47.102 INFO:teuthology.orchestra.run.smithi150.stdout:alertmanager.a smithi150 *:9093,9094 running (4m) 21s ago 8m 27.5M - 0.25.0 c8568f914cd2 5c82bec44fb9 2024-04-05T02:57:47.102 INFO:teuthology.orchestra.run.smithi150.stdout:grafana.a smithi196 *:3000 error 27s ago 8m - - 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi150.jnrwgs smithi150 running (7m) 21s ago 7m 15.4M - 17.2.7-805-g6fda41ff a49113eacc58 cfa13d4de7a1 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi196.eohomm smithi196 running (7m) 27s ago 7m 16.5M - 17.2.7-805-g6fda41ff a49113eacc58 e3ce4d99c3da 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.x smithi196 *:8443,9283,8765 running (2m) 27s ago 13m 450M - 19.0.0-2013-g0028525c d8c35eaf3c25 7ca2051bb392 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.y smithi150 *:8443,9283,8765 running (3m) 21s ago 14m 498M - 19.0.0-2013-g0028525c d8c35eaf3c25 0d5b82cc2eb2 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mon.a smithi150 running (2m) 21s ago 14m 39.9M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 39df812ce046 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mon.b smithi196 running (13m) 27s ago 13m 52.6M 2048M 17.2.7-805-g6fda41ff a49113eacc58 6680dc817949 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:mon.c smithi150 starting - - - 2048M 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.a smithi150 *:9100 running (4m) 21s ago 8m 16.6M - 1.5.0 0da6a335fe13 dc5e73ff394f 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.b smithi196 *:9100 running (3m) 27s ago 8m 15.5M - 1.5.0 0da6a335fe13 0d264938c335 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.0 smithi150 running (12m) 21s ago 12m 374M 2480M 17.2.7-805-g6fda41ff a49113eacc58 a6b8a0140540 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.1 smithi150 running (12m) 21s ago 12m 322M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2c503b0a41b4 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.2 smithi150 running (11m) 21s ago 11m 449M 2480M 17.2.7-805-g6fda41ff a49113eacc58 fee7e3893826 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.3 smithi150 running (11m) 21s ago 11m 471M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5d887a680c09 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.4 smithi196 running (11m) 27s ago 11m 399M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b21803209415 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.5 smithi196 running (10m) 27s ago 10m 430M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5415e890a870 2024-04-05T02:57:47.103 INFO:teuthology.orchestra.run.smithi150.stdout:osd.6 smithi196 running (10m) 27s ago 10m 296M 2480M 17.2.7-805-g6fda41ff a49113eacc58 85a4b25b62cf 2024-04-05T02:57:47.104 INFO:teuthology.orchestra.run.smithi150.stdout:osd.7 smithi196 running (9m) 27s ago 9m 373M 2480M 17.2.7-805-g6fda41ff a49113eacc58 ed60a8dad4f7 2024-04-05T02:57:47.104 INFO:teuthology.orchestra.run.smithi150.stdout:prometheus.a smithi196 *:9095 running (2m) 27s ago 9m 57.8M - 2.43.0 a07b618ecd1d dc41e7016bef 2024-04-05T02:57:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:47 smithi196 ceph-mon[33169]: pgmap v61: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-04-05T02:57:48.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:47 smithi196 ceph-mon[33169]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:48.170 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:47 smithi150 ceph-mon[103445]: pgmap v61: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-04-05T02:57:48.170 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:47 smithi150 ceph-mon[103445]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:48.170 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:47 smithi150 ceph-mon[107701]: pgmap v61: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-04-05T02:57:48.170 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:47 smithi150 ceph-mon[107701]: from='client.44099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:48.531 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T02:57:49.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:57:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:57:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:49.629 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:50.010 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:49 smithi150 ceph-mon[107701]: pgmap v62: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-04-05T02:57:50.010 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:49 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:50.010 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:49 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:50.011 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:49 smithi150 ceph-mon[103445]: pgmap v62: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-04-05T02:57:50.011 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:49 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:50.011 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:49 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:50.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:49 smithi196 ceph-mon[33169]: pgmap v62: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2024-04-05T02:57:50.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:49 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:50.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:49 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:51.953 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-mon[107701]: pgmap v63: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-04-05T02:57:51.953 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:51.953 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:51.954 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:57:51] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.43.0" 2024-04-05T02:57:51.954 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-mon[103445]: pgmap v63: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-04-05T02:57:51.954 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:51.954 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:51 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:52.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:51 smithi196 ceph-mon[33169]: pgmap v63: 89 pgs: 89 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-04-05T02:57:52.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:51 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:52.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:51 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:52.166 INFO:teuthology.orchestra.run.smithi150.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:52.877 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'sleep 60' 2024-04-05T02:57:53.337 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:53 smithi150 ceph-mon[107701]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:53.337 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:53 smithi150 ceph-mon[107701]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:53.337 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:53 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:53.338 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:53 smithi150 ceph-mon[103445]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:53.338 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:53 smithi150 ceph-mon[103445]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:53.338 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:53 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:53 smithi196 ceph-mon[33169]: from='client.44105 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:57:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:53 smithi196 ceph-mon[33169]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:57:53.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:53 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:53.914 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:57:54.341 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:54 smithi150 ceph-mon[107701]: pgmap v64: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-04-05T02:57:54.341 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:54 smithi150 ceph-mon[103445]: pgmap v64: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-04-05T02:57:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:54 smithi196 ceph-mon[33169]: pgmap v64: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 990 op/s 2024-04-05T02:57:56.424 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:56 smithi150 ceph-mon[107701]: pgmap v65: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2024-04-05T02:57:56.425 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:56.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:56 smithi150 ceph-mon[103445]: pgmap v65: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2024-04-05T02:57:56.425 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:56 smithi196 ceph-mon[33169]: pgmap v65: 89 pgs: 89 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 652 op/s 2024-04-05T02:57:56.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:56 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:57:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:58 smithi150 ceph-mon[103445]: pgmap v66: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-04-05T02:57:58.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:58 smithi150 ceph-mon[107701]: pgmap v66: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-04-05T02:57:58.438 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:58 smithi196 ceph-mon[33169]: pgmap v66: 89 pgs: 89 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-04-05T02:57:58.886 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:57:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:57:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:59.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:57:59 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:59.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.981 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.981 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.981 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:59.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:59.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:57:59.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:57:59.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:57:59.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:57:59.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:57:59 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:58:00.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:00 smithi196 ceph-mon[33169]: pgmap v67: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-04-05T02:58:00.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:00 smithi196 ceph-mon[33169]: pgmap v68: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2024-04-05T02:58:00.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:00 smithi196 ceph-mon[33169]: pgmap v69: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-04-05T02:58:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:00 smithi150 ceph-mon[107701]: pgmap v67: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-04-05T02:58:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:00 smithi150 ceph-mon[107701]: pgmap v68: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2024-04-05T02:58:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:00 smithi150 ceph-mon[107701]: pgmap v69: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-04-05T02:58:00.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:00 smithi150 ceph-mon[103445]: pgmap v67: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-04-05T02:58:00.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:00 smithi150 ceph-mon[103445]: pgmap v68: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2024-04-05T02:58:00.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:00 smithi150 ceph-mon[103445]: pgmap v69: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 988 op/s 2024-04-05T02:58:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:01 smithi196 ceph-mon[33169]: pgmap v70: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 500 op/s 2024-04-05T02:58:01.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:01 smithi150 ceph-mon[103445]: pgmap v70: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 500 op/s 2024-04-05T02:58:01.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:58:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:58:01] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2024-04-05T02:58:01.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:01 smithi150 ceph-mon[107701]: pgmap v70: 89 pgs: 89 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 500 op/s 2024-04-05T02:58:03.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:03 smithi150 ceph-mon[103445]: pgmap v71: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-05T02:58:03.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:03 smithi150 ceph-mon[107701]: pgmap v71: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-05T02:58:04.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:03 smithi196 ceph-mon[33169]: pgmap v71: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-04-05T02:58:06.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:05 smithi196 ceph-mon[33169]: pgmap v72: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-04-05T02:58:06.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:05 smithi150 ceph-mon[103445]: pgmap v72: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-04-05T02:58:06.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:05 smithi150 ceph-mon[107701]: pgmap v72: 89 pgs: 89 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2024-04-05T02:58:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:07 smithi196 ceph-mon[33169]: pgmap v73: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-04-05T02:58:08.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:07 smithi150 ceph-mon[107701]: pgmap v73: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-04-05T02:58:08.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:07 smithi150 ceph-mon[103445]: pgmap v73: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-04-05T02:58:09.136 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:58:08 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:58:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:58:10.029 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:09 smithi150 ceph-mon[107701]: pgmap v74: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-04-05T02:58:10.029 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:09 smithi150 ceph-mon[103445]: pgmap v74: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-04-05T02:58:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:09 smithi196 ceph-mon[33169]: pgmap v74: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-04-05T02:58:10.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:10 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:10.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:10 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:10 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:11.946 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:11 smithi150 ceph-mon[107701]: pgmap v75: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-04-05T02:58:11.946 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:58:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:58:11] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2024-04-05T02:58:11.947 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:11 smithi150 ceph-mon[103445]: pgmap v75: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-04-05T02:58:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:11 smithi196 ceph-mon[33169]: pgmap v75: 89 pgs: 89 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 740 op/s 2024-04-05T02:58:14.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:13 smithi196 ceph-mon[33169]: pgmap v76: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:13 smithi150 ceph-mon[107701]: pgmap v76: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:13 smithi150 ceph-mon[103445]: pgmap v76: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:16.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:15 smithi196 ceph-mon[33169]: pgmap v77: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-04-05T02:58:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:15 smithi150 ceph-mon[107701]: pgmap v77: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-04-05T02:58:16.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:15 smithi150 ceph-mon[103445]: pgmap v77: 89 pgs: 89 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-04-05T02:58:18.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:17 smithi196 ceph-mon[33169]: pgmap v78: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-05T02:58:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:17 smithi150 ceph-mon[107701]: pgmap v78: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-05T02:58:18.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:17 smithi150 ceph-mon[103445]: pgmap v78: 89 pgs: 89 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-04-05T02:58:19.137 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:58:18 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:58:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:58:20.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:19 smithi196 ceph-mon[33169]: pgmap v79: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2024-04-05T02:58:20.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:19 smithi150 ceph-mon[107701]: pgmap v79: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2024-04-05T02:58:20.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:19 smithi150 ceph-mon[103445]: pgmap v79: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 903 op/s 2024-04-05T02:58:21.784 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:58:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:58:21] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2024-04-05T02:58:22.054 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:21 smithi150 ceph-mon[103445]: pgmap v80: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-04-05T02:58:22.054 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:21 smithi150 ceph-mon[107701]: pgmap v80: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-04-05T02:58:22.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:21 smithi196 ceph-mon[33169]: pgmap v80: 89 pgs: 89 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 717 op/s 2024-04-05T02:58:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:23 smithi196 ceph-mon[33169]: pgmap v81: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:24.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:23 smithi150 ceph-mon[103445]: pgmap v81: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:23 smithi150 ceph-mon[107701]: pgmap v81: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:25 smithi196 ceph-mon[33169]: pgmap v82: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-05T02:58:26.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:25 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:25 smithi150 ceph-mon[103445]: pgmap v82: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-05T02:58:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:25 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:25 smithi150 ceph-mon[107701]: pgmap v82: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-05T02:58:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:25 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:28.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:27 smithi196 ceph-mon[33169]: pgmap v83: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-04-05T02:58:28.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:27 smithi150 ceph-mon[103445]: pgmap v83: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-04-05T02:58:28.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:27 smithi150 ceph-mon[107701]: pgmap v83: 89 pgs: 89 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-04-05T02:58:29.038 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 02:58:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T02:58:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi196.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)\", cluster_addr=\"172.21.15.150\", device_class=\"ssd\", hostname=\"smithi150\", instance=\"smithi150:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.150\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-05T02:58:30.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:29 smithi196 ceph-mon[33169]: pgmap v84: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-04-05T02:58:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:29 smithi150 ceph-mon[103445]: pgmap v84: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-04-05T02:58:30.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:29 smithi150 ceph-mon[107701]: pgmap v84: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 920 op/s 2024-04-05T02:58:31.808 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:58:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:58:31] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2024-04-05T02:58:32.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:31 smithi196 ceph-mon[33169]: pgmap v85: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-05T02:58:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:31 smithi150 ceph-mon[107701]: pgmap v85: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-05T02:58:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:31 smithi150 ceph-mon[103445]: pgmap v85: 89 pgs: 89 active+clean; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 737 op/s 2024-04-05T02:58:34.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:33 smithi196 ceph-mon[33169]: pgmap v86: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-05T02:58:34.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:33 smithi150 ceph-mon[103445]: pgmap v86: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-05T02:58:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:33 smithi150 ceph-mon[107701]: pgmap v86: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-04-05T02:58:36.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:35 smithi196 ceph-mon[33169]: pgmap v87: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-05T02:58:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:35 smithi150 ceph-mon[103445]: pgmap v87: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-05T02:58:36.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:35 smithi150 ceph-mon[107701]: pgmap v87: 89 pgs: 89 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-05T02:58:38.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:37 smithi196 ceph-mon[33169]: pgmap v88: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-04-05T02:58:38.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:37 smithi150 ceph-mon[107701]: pgmap v88: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-04-05T02:58:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:37 smithi150 ceph-mon[103445]: pgmap v88: 89 pgs: 89 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 850 op/s 2024-04-05T02:58:40.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:39 smithi196 ceph-mon[33169]: pgmap v89: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-04-05T02:58:40.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:39 smithi150 ceph-mon[103445]: pgmap v89: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-04-05T02:58:40.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:39 smithi150 ceph-mon[107701]: pgmap v89: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 985 op/s 2024-04-05T02:58:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:40.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:41.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:40 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:41.840 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:58:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:58:41] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2024-04-05T02:58:42.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:41 smithi196 ceph-mon[33169]: pgmap v90: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-05T02:58:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:41 smithi150 ceph-mon[103445]: pgmap v90: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-05T02:58:42.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:41 smithi150 ceph-mon[107701]: pgmap v90: 89 pgs: 89 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-05T02:58:44.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:43 smithi196 ceph-mon[33169]: pgmap v91: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:44.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:43 smithi150 ceph-mon[103445]: pgmap v91: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:44.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:43 smithi150 ceph-mon[107701]: pgmap v91: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:46.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:45 smithi196 ceph-mon[33169]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-05T02:58:46.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:45 smithi150 ceph-mon[103445]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-05T02:58:46.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:45 smithi150 ceph-mon[107701]: pgmap v92: 89 pgs: 89 active+clean; 2.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 720 op/s 2024-04-05T02:58:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:47 smithi196 ceph-mon[33169]: pgmap v93: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-04-05T02:58:48.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:47 smithi150 ceph-mon[107701]: pgmap v93: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-04-05T02:58:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:47 smithi150 ceph-mon[103445]: pgmap v93: 89 pgs: 89 active+clean; 2.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-04-05T02:58:50.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:49 smithi196 ceph-mon[33169]: pgmap v94: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-05T02:58:50.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:49 smithi150 ceph-mon[107701]: pgmap v94: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-05T02:58:50.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:49 smithi150 ceph-mon[103445]: pgmap v94: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-04-05T02:58:51.865 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:58:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:58:51] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2024-04-05T02:58:52.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:51 smithi196 ceph-mon[33169]: pgmap v95: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-05T02:58:52.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:51 smithi150 ceph-mon[107701]: pgmap v95: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-05T02:58:52.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:51 smithi150 ceph-mon[103445]: pgmap v95: 89 pgs: 89 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 721 op/s 2024-04-05T02:58:54.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:53 smithi196 ceph-mon[33169]: pgmap v96: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:53 smithi150 ceph-mon[107701]: pgmap v96: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:54.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:53 smithi150 ceph-mon[103445]: pgmap v96: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-04-05T02:58:56.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:55 smithi196 ceph-mon[33169]: pgmap v97: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-04-05T02:58:56.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:55 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:55 smithi150 ceph-mon[107701]: pgmap v97: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-04-05T02:58:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:55 smithi150 ceph-mon[103445]: pgmap v97: 89 pgs: 89 active+clean; 1.7 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 735 op/s 2024-04-05T02:58:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:58:57.538 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade resume' 2024-04-05T02:58:58.006 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:58:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:57 smithi150 ceph-mon[103445]: pgmap v98: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-04-05T02:58:58.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:57 smithi150 ceph-mon[107701]: pgmap v98: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-04-05T02:58:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:57 smithi196 ceph-mon[33169]: pgmap v98: 89 pgs: 89 active+clean; 1.7 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 780 op/s 2024-04-05T02:58:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:58 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:59.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:58 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:58:59.287 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:58 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:00.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:58:59 smithi150 ceph-mon[103445]: pgmap v99: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-04-05T02:59:00.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:58:59 smithi150 ceph-mon[107701]: pgmap v99: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-04-05T02:59:00.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:58:59 smithi196 ceph-mon[33169]: pgmap v99: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-04-05T02:59:00.534 INFO:teuthology.orchestra.run.smithi150.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:01.528 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for all mons to upgrade' 2024-04-05T02:59:01.753 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-mon[103445]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:01.754 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-mon[103445]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:01.754 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:01.754 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:59:01] "GET /metrics HTTP/1.1" 200 36362 "" "Prometheus/2.43.0" 2024-04-05T02:59:01.754 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-mon[107701]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:01.754 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-mon[107701]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:01.754 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:01 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:01.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:01 smithi196 ceph-mon[33169]: from='client.44111 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:01.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:01 smithi196 ceph-mon[33169]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:01.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:01 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:02.071 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:02.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:02 smithi196 ceph-mon[33169]: pgmap v100: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-05T02:59:02.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:02 smithi150 ceph-mon[107701]: pgmap v100: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-05T02:59:02.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:02 smithi150 ceph-mon[103445]: pgmap v100: 89 pgs: 89 active+clean; 1.5 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 733 op/s 2024-04-05T02:59:03.803 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:03 smithi150 ceph-mon[107701]: pgmap v101: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:59:03.804 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:03 smithi150 ceph-mon[103445]: pgmap v101: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:59:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:03 smithi196 ceph-mon[33169]: pgmap v101: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-04-05T02:59:04.893 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:04.893 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:04.893 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:04.894 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:04 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:04 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:05.322 INFO:teuthology.orchestra.run.smithi150.stdout:wait for all mons to upgrade 2024-04-05T02:59:06.026 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:05 smithi150 ceph-mon[107701]: pgmap v102: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:59:06.027 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:05 smithi150 ceph-mon[103445]: pgmap v102: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:59:06.071 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-05T02:59:06.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:05 smithi196 ceph-mon[33169]: pgmap v102: 89 pgs: 89 active+clean; 1.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s 2024-04-05T02:59:07.283 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:08.013 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:07 smithi150 ceph-mon[107701]: pgmap v103: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-05T02:59:08.013 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:07 smithi150 ceph-mon[103445]: pgmap v103: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-05T02:59:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:07 smithi196 ceph-mon[33169]: pgmap v103: 89 pgs: 89 active+clean; 1.2 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 734 op/s 2024-04-05T02:59:09.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:09 smithi150 ceph-mon[103445]: pgmap v104: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2024-04-05T02:59:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:09 smithi150 ceph-mon[107701]: pgmap v104: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2024-04-05T02:59:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:09 smithi196 ceph-mon[33169]: pgmap v104: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2024-04-05T02:59:11.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:59:11] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/569155009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:11.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:11 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/569155009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:11 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:11 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:11 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/569155009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: pgmap v105: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: pgmap v106: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: pgmap v107: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 449 op/s 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:12.749 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:12.750 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:59:12.750 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:59:12.750 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:12 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-05T02:59:12.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: pgmap v105: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: pgmap v106: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: pgmap v107: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 449 op/s 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:59:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: pgmap v105: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 656 op/s 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: pgmap v106: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: pgmap v107: 89 pgs: 89 active+clean; 1014 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 449 op/s 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-05T02:59:12.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-05T02:59:13.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:13 smithi196 ceph-mon[33169]: Upgrade: It appears safe to stop mon.b 2024-04-05T02:59:13.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:13 smithi150 ceph-mon[107701]: Upgrade: It appears safe to stop mon.b 2024-04-05T02:59:13.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:13 smithi150 ceph-mon[103445]: Upgrade: It appears safe to stop mon.b 2024-04-05T02:59:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:14 smithi196 ceph-mon[33169]: pgmap v108: 89 pgs: 89 active+clean; 764 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-04-05T02:59:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:14 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/3880661907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:14 smithi196 ceph-mon[33169]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:14 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:14 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:14.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:14 smithi196 ceph-mon[33169]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[103445]: pgmap v108: 89 pgs: 89 active+clean; 764 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3880661907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[107701]: pgmap v108: 89 pgs: 89 active+clean; 764 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-04-05T02:59:14.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3880661907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:14.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:14.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:14.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:14.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:14 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:15.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:15 smithi196 ceph-mon[33169]: Upgrade: Updating mon.b 2024-04-05T02:59:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:15 smithi196 ceph-mon[33169]: Deploying daemon mon.b on smithi196 2024-04-05T02:59:15.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:15 smithi196 ceph-mon[33169]: pgmap v109: 89 pgs: 89 active+clean; 764 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2024-04-05T02:59:15.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:15 smithi150 ceph-mon[103445]: Upgrade: Updating mon.b 2024-04-05T02:59:15.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:15 smithi150 ceph-mon[103445]: Deploying daemon mon.b on smithi196 2024-04-05T02:59:15.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:15 smithi150 ceph-mon[103445]: pgmap v109: 89 pgs: 89 active+clean; 764 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2024-04-05T02:59:15.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:15 smithi150 ceph-mon[107701]: Upgrade: Updating mon.b 2024-04-05T02:59:15.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:15 smithi150 ceph-mon[107701]: Deploying daemon mon.b on smithi196 2024-04-05T02:59:15.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:15 smithi150 ceph-mon[107701]: pgmap v109: 89 pgs: 89 active+clean; 764 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2024-04-05T02:59:17.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:16 smithi196 systemd[1]: Stopping Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:59:17.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:16 smithi196 ceph-mon[33169]: from='client.? 172.21.15.150:0/4176436927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:17.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:16 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4176436927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:17.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:16 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4176436927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:17 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b[33146]: 2024-04-05T02:59:17.369+0000 7f20a425b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T02:59:17.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:17 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b[33146]: 2024-04-05T02:59:17.369+0000 7f20a425b700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-05T02:59:17.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:17 smithi196 podman[91172]: 2024-04-05 02:59:17.516411396 +0000 UTC m=+0.628466435 container died 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T02:59:17.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:59:17] ENGINE Bus STOPPING 2024-04-05T02:59:17.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:59:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T02:59:17.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:59:17] ENGINE Bus STOPPED 2024-04-05T02:59:17.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:59:17] ENGINE Bus STARTING 2024-04-05T02:59:17.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:59:17] ENGINE Serving on http://:::9283 2024-04-05T02:59:17.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:17 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:02:59:17] ENGINE Bus STARTED 2024-04-05T02:59:18.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:17 smithi196 podman[91172]: 2024-04-05 02:59:17.951037029 +0000 UTC m=+1.063092071 container cleanup 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2) 2024-04-05T02:59:18.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:17 smithi196 bash[91172]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b 2024-04-05T02:59:18.551 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:18 smithi196 podman[91191]: 2024-04-05 02:59:18.186200987 +0000 UTC m=+0.663882245 container remove 6680dc8179495bc8cb4f25ab35d5c87ffa70b2e1eb83076e42dc2204f5622e0c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-05T02:59:18.855 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:18 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.b.service: Deactivated successfully. 2024-04-05T02:59:18.855 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:18 smithi196 systemd[1]: Stopped Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:59:18.856 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:18 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.b.service: Consumed 11.973s CPU time. 2024-04-05T02:59:19.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:18 smithi196 systemd[1]: Starting Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T02:59:19.416 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 podman[91314]: 2024-04-05 02:59:19.045600936 +0000 UTC m=+0.026844238 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:19.416 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 podman[91314]: 2024-04-05 02:59:19.170818516 +0000 UTC m=+0.152061812 container create 51bbb41f1c91054951f544fa05101ef046fba67950e1489e1d413442cf14a6b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T02:59:19.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 podman[91314]: 2024-04-05 02:59:19.416251174 +0000 UTC m=+0.397494484 container init 51bbb41f1c91054951f544fa05101ef046fba67950e1489e1d413442cf14a6b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-05T02:59:19.892 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 podman[91314]: 2024-04-05 02:59:19.42443006 +0000 UTC m=+0.405673352 container start 51bbb41f1c91054951f544fa05101ef046fba67950e1489e1d413442cf14a6b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: set uid:gid to 167:167 (ceph:ceph) 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev), process ceph-mon, pid 7 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: pidfile_write: ignore empty --pid-file 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: load: jerasure load: lrc 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: RocksDB version: 7.9.2 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Git sha 0 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Compile date 2024-04-04 02:01:22 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: DB SUMMARY 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: DB Session ID: 3O4SF4SAKQUVAGULB5CS 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: CURRENT file: CURRENT 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: IDENTITY file: IDENTITY 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1092 Bytes 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 9468402 ; 2024-04-05T02:59:19.893 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.error_if_exists: 0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.create_if_missing: 0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.paranoid_checks: 1 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.env: 0x56235febbc60 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.fs: PosixFileSystem 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.info_log: 0x5623611f5ae0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_file_opening_threads: 16 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.statistics: (nil) 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.use_fsync: 0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_log_file_size: 0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-05T02:59:19.894 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.keep_log_file_num: 1000 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.recycle_log_file_num: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.allow_fallocate: 1 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.allow_mmap_reads: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.allow_mmap_writes: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.use_direct_reads: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.create_missing_column_families: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.db_log_dir: 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.wal_dir: 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-05T02:59:19.895 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.advise_random_on_open: 1 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.db_write_buffer_size: 0 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.write_buffer_manager: 0x5623611c8820 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.rate_limiter: (nil) 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.wal_recovery_mode: 2 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.enable_thread_tracking: 0 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.enable_pipelined_write: 0 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.unordered_write: 0 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-05T02:59:19.896 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.row_cache: None 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.wal_filter: None 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.allow_ingest_behind: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.two_write_queues: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.manual_wal_flush: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.wal_compression: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.atomic_flush: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.log_readahead_size: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.best_efforts_recovery: 0 2024-04-05T02:59:19.897 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.allow_data_in_errors: 0 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.db_host_id: __hostname__ 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_background_jobs: 2 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_background_compactions: -1 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_subcompactions: 1 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_total_wal_size: 0 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-05T02:59:19.898 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_open_files: -1 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bytes_per_sync: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_readahead_size: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_background_flushes: -1 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Compression algorithms supported: 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kZSTD supported: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kXpressCompression supported: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kLZ4HCCompression supported: 1 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kZlibCompression supported: 1 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kSnappyCompression supported: 1 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kLZ4Compression supported: 1 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: kBZip2Compression supported: 0 2024-04-05T02:59:19.899 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.merge_operator: 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_filter: None 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_filter_factory: None 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.sst_partitioner_factory: None 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5623611f5be0) 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout: cache_index_and_filter_blocks: 1 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout: pin_top_level_index_and_filter: 1 2024-04-05T02:59:19.900 INFO:journalctl@ceph.mon.b.smithi196.stdout: index_type: 0 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: data_block_index_type: 0 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: index_shortening: 1 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: checksum: 4 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: no_block_cache: 0 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache: 0x5623611f9090 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache_name: BinnedLRUCache 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache_options: 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: capacity : 536870912 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: num_shard_bits : 4 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: strict_capacity_limit : 0 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: high_pri_pool_ratio: 0.000 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_cache_compressed: (nil) 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: persistent_cache: (nil) 2024-04-05T02:59:19.901 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_size: 4096 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_size_deviation: 10 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_restart_interval: 16 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: index_block_restart_interval: 1 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: metadata_block_size: 4096 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: partition_filters: 0 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: use_delta_encoding: 1 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: filter_policy: bloomfilter 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: whole_key_filtering: 1 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: verify_compression: 0 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: read_amp_bytes_per_bit: 0 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: format_version: 5 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: enable_index_compression: 1 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: block_align: 0 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: max_auto_readahead_size: 262144 2024-04-05T02:59:19.902 INFO:journalctl@ceph.mon.b.smithi196.stdout: prepopulate_block_cache: 0 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout: initial_auto_readahead_size: 8192 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout: num_file_reads_for_auto_readahead: 2 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.write_buffer_size: 33554432 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_write_buffer_number: 2 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression: NoCompression 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression: Disabled 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.prefix_extractor: nullptr 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.num_levels: 7 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-05T02:59:19.903 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.level: 32767 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.strategy: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.enabled: false 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-05T02:59:19.904 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.target_file_size_base: 67108864 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-05T02:59:19.905 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.arena_block_size: 1048576 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.disable_auto_compactions: 0 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-05T02:59:19.906 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.table_properties_collectors: 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.inplace_update_support: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.bloom_locality: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.max_successive_merges: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.paranoid_file_checks: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.force_consistency_checks: 1 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.report_bg_io_stats: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.ttl: 2592000 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-05T02:59:19.907 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.enable_blob_files: false 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.min_blob_size: 0 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.blob_file_size: 268435456 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.blob_file_starting_level: 0 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12751, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-05T02:59:19.908 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 64bb1cb2-7c84-45f0-9f38-0bc83ad7c3a4 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285959469404, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285959506133, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 5799605, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12756, "largest_seqno": 16636, "table_properties": {"data_size": 5784318, "index_size": 10170, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4101, "raw_key_size": 34418, "raw_average_key_size": 21, "raw_value_size": 5751689, "raw_average_value_size": 3565, "num_data_blocks": 456, "num_entries": 1613, "num_filter_entries": 1613, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712285959, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "64bb1cb2-7c84-45f0-9f38-0bc83ad7c3a4", "db_session_id": "3O4SF4SAKQUVAGULB5CS", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712285959506284, "job": 1, "event": "recovery_finished"} 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5623612e6000 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: DB pointer 0x5623612d4000 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** DB Stats ** 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:59:19.909 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** Compaction Stats [default] ** 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: L0 1/0 5.53 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 410.6 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: L6 1/0 6.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Sum 2/0 11.88 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 410.6 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 410.6 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** Compaction Stats [default] ** 2024-04-05T02:59:19.910 INFO:journalctl@ceph.mon.b.smithi196.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 410.6 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Cumulative compaction: 0.01 GB write, 125.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Interval compaction: 0.01 GB write, 125.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-05T02:59:19.911 INFO:journalctl@ceph.mon.b.smithi196.stdout: Block cache BinnedLRUCache@0x5623611f9090#7 capacity: 512.00 MB usage: 68.53 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: Block cache entry stats(count,size,portion): DataBlock(4,28.81 KB,0.00549555%) FilterBlock(2,13.56 KB,0.00258684%) IndexBlock(2,26.16 KB,0.00498891%) Misc(1,0.00 KB,0%) 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: starting mon.b rank 1 at public addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] at bind addrs [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???) e3 preinit fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).mds e9 new map 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).mds e9 print_map 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: e9 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: legacy client fscid: 1 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: Filesystem 'foo' (1) 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: fs_name foo 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: epoch 8 2024-04-05T02:59:19.912 INFO:journalctl@ceph.mon.b.smithi196.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: created 2024-04-05T02:49:47.190260+0000 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: modified 2024-04-05T02:50:03.911335+0000 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: tableserver 0 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: root 0 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: session_timeout 60 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: session_autoclose 300 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: max_file_size 1099511627776 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: max_xattr_size 65536 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: required_client_features {} 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: last_failure 0 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: last_failure_osd_epoch 0 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: max_mds 1 2024-04-05T02:59:19.913 INFO:journalctl@ceph.mon.b.smithi196.stdout: in 0 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: up {0=24557} 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: failed 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: damaged 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: stopped 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: data_pools [4] 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: metadata_pool 3 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: inline_data disabled 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: balancer 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: bal_rank_mask -1 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: standby_count_wanted 1 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: qdb_cluster leader: 0 members: 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: [mds.foo.smithi150.jnrwgs{0:24557} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.150:6834/1439318202,v1:172.21.15.150:6835/1439318202] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.914 INFO:journalctl@ceph.mon.b.smithi196.stdout: Standby daemons: 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout: 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout: [mds.foo.smithi196.eohomm{-1:24526} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.196:6832/2559423220,v1:172.21.15.196:6833/2559423220] compat {c=[1],r=[1],i=[7ff]}] 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).osd e83 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).osd e83 crush map has features 432629239337189376, adjusting msgr requires 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 ceph-mon[91352]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 bash[91314]: 51bbb41f1c91054951f544fa05101ef046fba67950e1489e1d413442cf14a6b5 2024-04-05T02:59:19.915 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:19 smithi196 systemd[1]: Started Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T02:59:21.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: mon.b calling monitor election 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: mon.a calling monitor election 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: mon.c calling monitor election 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: monmap epoch 4 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: last_changed 2024-04-05T02:59:20.401024+0000 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: min_mon_release 19 (squid) 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: election_strategy: 1 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:59:21.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: daemon grafana.a on smithi196 is in error state 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='' 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:21.730 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:59:21] "GET /metrics HTTP/1.1" 200 36361 "" "Prometheus/2.43.0" 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: mon.b calling monitor election 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: mon.a calling monitor election 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: mon.c calling monitor election 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: monmap epoch 4 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: last_changed 2024-04-05T02:59:20.401024+0000 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: min_mon_release 19 (squid) 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: election_strategy: 1 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:59:21.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: daemon grafana.a on smithi196 is in error state 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='' 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:21.732 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-05T02:59:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-05T02:59:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-05T02:59:21.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: mon.b calling monitor election 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: mon.a calling monitor election 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: mon.c calling monitor election 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: monmap epoch 4 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: last_changed 2024-04-05T02:59:20.401024+0000 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: created 2024-04-05T02:42:41.704408+0000 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: min_mon_release 19 (squid) 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: election_strategy: 1 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: 0: [v2:172.21.15.150:3300/0,v1:172.21.15.150:6789/0] mon.a 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: 1: [v2:172.21.15.196:3300/0,v1:172.21.15.196:6789/0] mon.b 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: 2: [v2:172.21.15.150:3301/0,v1:172.21.15.150:6790/0] mon.c 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: fsmap foo:1 {0=foo.smithi150.jnrwgs=up:active} 1 up:standby 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: osdmap e83: 8 total, 8 up, 8 in 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: mgrmap e40: y(active, since 3m), standbys: x 2024-04-05T02:59:21.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T02:59:21.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-04-05T02:59:21.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: daemon grafana.a on smithi196 is in error state 2024-04-05T02:59:21.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='' 2024-04-05T02:59:21.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:21.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T02:59:22.259 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 3 2024-04-05T02:59:22.611 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:22 smithi150 ceph-mon[103445]: pgmap v112: 89 pgs: 89 active+clean; 520 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2024-04-05T02:59:22.611 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/334002083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:22.611 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:22 smithi150 ceph-mon[107701]: pgmap v112: 89 pgs: 89 active+clean; 520 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2024-04-05T02:59:22.612 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/334002083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:22.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:22 smithi196 ceph-mon[91352]: pgmap v112: 89 pgs: 89 active+clean; 520 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 790 op/s 2024-04-05T02:59:22.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/334002083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-05T02:59:23.062 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T02:59:23.523 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:23.850 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:23 smithi150 ceph-mon[103445]: pgmap v113: 89 pgs: 89 active+clean; 256 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:59:23.850 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:23 smithi150 ceph-mon[107701]: pgmap v113: 89 pgs: 89 active+clean; 256 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:59:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:23 smithi196 ceph-mon[91352]: pgmap v113: 89 pgs: 89 active+clean; 256 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-04-05T02:59:24.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T02:59:24.523+0000 7f45dfa59700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-05T02:59:25.856 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:25 smithi196 ceph-mon[91352]: pgmap v114: 89 pgs: 89 active+clean; 256 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-04-05T02:59:25.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:25 smithi150 ceph-mon[103445]: pgmap v114: 89 pgs: 89 active+clean; 256 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-04-05T02:59:25.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:25 smithi150 ceph-mon[107701]: pgmap v114: 89 pgs: 89 active+clean; 256 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 676 op/s 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:alertmanager.a smithi150 *:9093,9094 running (6m) 15s ago 10m 27.5M - 0.25.0 c8568f914cd2 5c82bec44fb9 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:grafana.a smithi196 *:3000 error 22s ago 10m - - 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi150.jnrwgs smithi150 running (9m) 15s ago 9m 16.2M - 17.2.7-805-g6fda41ff a49113eacc58 cfa13d4de7a1 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi196.eohomm smithi196 running (9m) 22s ago 9m 16.8M - 17.2.7-805-g6fda41ff a49113eacc58 e3ce4d99c3da 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.x smithi196 *:8443,9283,8765 running (4m) 22s ago 14m 451M - 19.0.0-2013-g0028525c d8c35eaf3c25 7ca2051bb392 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.y smithi150 *:8443,9283,8765 running (5m) 15s ago 16m 502M - 19.0.0-2013-g0028525c d8c35eaf3c25 0d5b82cc2eb2 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mon.a smithi150 running (3m) 15s ago 16m 44.5M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 39df812ce046 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mon.b smithi196 starting - - - 2048M 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:mon.c smithi150 running (112s) 15s ago 14m 34.4M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 74316b4d98ae 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.a smithi150 *:9100 running (6m) 15s ago 10m 17.1M - 1.5.0 0da6a335fe13 dc5e73ff394f 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.b smithi196 *:9100 running (5m) 22s ago 10m 15.4M - 1.5.0 0da6a335fe13 0d264938c335 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:osd.0 smithi150 running (14m) 15s ago 14m 387M 2480M 17.2.7-805-g6fda41ff a49113eacc58 a6b8a0140540 2024-04-05T02:59:26.220 INFO:teuthology.orchestra.run.smithi150.stdout:osd.1 smithi150 running (13m) 15s ago 13m 338M 2480M 17.2.7-805-g6fda41ff a49113eacc58 2c503b0a41b4 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:osd.2 smithi150 running (13m) 15s ago 13m 470M 2480M 17.2.7-805-g6fda41ff a49113eacc58 fee7e3893826 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:osd.3 smithi150 running (13m) 15s ago 13m 490M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5d887a680c09 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:osd.4 smithi196 running (12m) 22s ago 12m 411M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b21803209415 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:osd.5 smithi196 running (12m) 22s ago 12m 440M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5415e890a870 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:osd.6 smithi196 running (11m) 22s ago 11m 311M 2480M 17.2.7-805-g6fda41ff a49113eacc58 85a4b25b62cf 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:osd.7 smithi196 running (11m) 22s ago 11m 386M 2480M 17.2.7-805-g6fda41ff a49113eacc58 ed60a8dad4f7 2024-04-05T02:59:26.221 INFO:teuthology.orchestra.run.smithi150.stdout:prometheus.a smithi196 *:9095 running (4m) 22s ago 11m 62.2M - 2.43.0 a07b618ecd1d dc41e7016bef 2024-04-05T02:59:26.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:26.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[103445]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:26.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:26.882 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:26.882 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:26.882 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[107701]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:26.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:26.883 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:26 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:26.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:26.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:26 smithi196 ceph-mon[91352]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:26.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:26 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:26.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:26 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:26.924 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T02:59:26.965 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Removed 128281 objects 2024-04-05T02:59:26.965 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Clean up completed and total clean up time :151.372 2024-04-05T02:59:27.198 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:27.850 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:27 smithi150 ceph-mon[107701]: pgmap v115: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-05T02:59:27.850 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:27 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:27.850 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:27 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:27.850 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:27 smithi150 ceph-mon[103445]: pgmap v115: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-05T02:59:27.851 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:27 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:27.851 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:27 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:27 smithi196 ceph-mon[91352]: pgmap v115: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-04-05T02:59:27.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:27 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:27.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:27 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:29.577 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-04-05T02:59:29.577 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-04-05T02:59:29.701 INFO:teuthology.orchestra.run.smithi150.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:29.748 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:29 smithi196 ceph-mon[91352]: pgmap v116: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:59:29.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:29 smithi150 ceph-mon[103445]: pgmap v116: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:59:29.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:29 smithi150 ceph-mon[107701]: pgmap v116: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 679 op/s 2024-04-05T02:59:30.902 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'sleep 60' 2024-04-05T02:59:30.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:30 smithi196 ceph-mon[91352]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:30.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:30 smithi196 ceph-mon[91352]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:30.969 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:30 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:30.975 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:30.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:30 smithi150 ceph-mon[103445]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:30.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:30 smithi150 ceph-mon[103445]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:30.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:30 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:30.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:30 smithi150 ceph-mon[107701]: from='client.34203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T02:59:30.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:30 smithi150 ceph-mon[107701]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T02:59:30.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:30 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:31.697 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:31.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:59:31] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-05T02:59:32.174 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.174 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: pgmap v117: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: pgmap v118: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: Reconfiguring mon.a (monmap changed)... 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[103445]: Reconfiguring daemon mon.a on smithi150 2024-04-05T02:59:32.175 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: pgmap v117: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: pgmap v118: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: Reconfiguring mon.a (monmap changed)... 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:32.176 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:32 smithi150 ceph-mon[107701]: Reconfiguring daemon mon.a on smithi150 2024-04-05T02:59:32.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: pgmap v117: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: pgmap v118: 89 pgs: 89 active+clean; 9.4 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: Reconfiguring mon.a (monmap changed)... 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:32.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:32 smithi196 ceph-mon[91352]: Reconfiguring daemon mon.a on smithi150 2024-04-05T02:59:34.337 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:34 smithi150 ceph-mon[103445]: pgmap v119: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-04-05T02:59:34.337 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:34 smithi150 ceph-mon[107701]: pgmap v119: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-04-05T02:59:34.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:34 smithi196 ceph-mon[91352]: pgmap v119: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-04-05T02:59:35.906 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_1' removed 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: Reconfiguring mgr.y (monmap changed)... 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:36.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: Reconfiguring daemon mgr.y on smithi150 2024-04-05T02:59:36.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-04-05T02:59:36.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2781721135' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:59:36.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:35 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:59:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: Reconfiguring mgr.y (monmap changed)... 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: Reconfiguring daemon mgr.y on smithi150 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2781721135' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:59:36.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: Reconfiguring mgr.y (monmap changed)... 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: Reconfiguring daemon mgr.y on smithi150 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: pgmap v120: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 428 op/s 2024-04-05T02:59:36.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2781721135' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:59:36.230 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:35 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T02:59:36.850 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-05T02:59:36.850 INFO:tasks.radosbench:Beginning radosbench... 2024-04-05T02:59:36.851 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-04-05T02:59:36.851 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool create unique_pool_2 16 2024-04-05T02:59:37.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T02:59:37.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[107701]: failed to encode map e84 with expected crc 2024-04-05T02:59:37.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[107701]: failed to encode map e84 with expected crc 2024-04-05T02:59:37.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[107701]: osdmap e84: 8 total, 8 up, 8 in 2024-04-05T02:59:37.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T02:59:37.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[103445]: failed to encode map e84 with expected crc 2024-04-05T02:59:37.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[103445]: failed to encode map e84 with expected crc 2024-04-05T02:59:37.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:36 smithi150 ceph-mon[103445]: osdmap e84: 8 total, 8 up, 8 in 2024-04-05T02:59:37.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:36 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T02:59:37.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:36 smithi196 ceph-mon[91352]: failed to encode map e84 with expected crc 2024-04-05T02:59:37.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:36 smithi196 ceph-mon[91352]: failed to encode map e84 with expected crc 2024-04-05T02:59:37.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:36 smithi196 ceph-mon[91352]: osdmap e84: 8 total, 8 up, 8 in 2024-04-05T02:59:37.440 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:38.536 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:38.536 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[103445]: Reconfiguring daemon mon.c on smithi150 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.537 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:38 smithi150 ceph-mon[107701]: Reconfiguring daemon mon.c on smithi150 2024-04-05T02:59:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: pgmap v122: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:38.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:38.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: Reconfiguring mon.c (monmap changed)... 2024-04-05T02:59:38.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:38.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:38.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:38.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:38 smithi196 ceph-mon[91352]: Reconfiguring daemon mon.c on smithi150 2024-04-05T02:59:40.370 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_2' created 2024-04-05T02:59:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:40 smithi196 ceph-mon[91352]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4022555545' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-05T02:59:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:59:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:40.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[103445]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:40.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4022555545' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-05T02:59:40.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:40.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:40.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:59:40.676 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:40.677 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[107701]: pgmap v123: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:40.677 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4022555545' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2024-04-05T02:59:40.677 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:40.677 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:40.677 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T02:59:40.677 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:41.010 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-04-05T02:59:41.575 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:59:41] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2024-04-05T02:59:41.575 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: Reconfiguring osd.0 (monmap changed)... 2024-04-05T02:59:41.575 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.0 on smithi150 2024-04-05T02:59:41.575 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4022555545' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-05T02:59:41.575 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: failed to encode map e85 with expected crc 2024-04-05T02:59:41.575 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: failed to encode map e85 with expected crc 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: osdmap e85: 8 total, 8 up, 8 in 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: Reconfiguring osd.0 (monmap changed)... 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.0 on smithi150 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4022555545' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: failed to encode map e85 with expected crc 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: failed to encode map e85 with expected crc 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: osdmap e85: 8 total, 8 up, 8 in 2024-04-05T02:59:41.576 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: Reconfiguring osd.0 (monmap changed)... 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.0 on smithi150 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4022555545' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: failed to encode map e85 with expected crc 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: failed to encode map e85 with expected crc 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: osdmap e85: 8 total, 8 up, 8 in 2024-04-05T02:59:41.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:41.807 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T02:59:42.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: pgmap v125: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: osdmap e86: 8 total, 8 up, 8 in 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:42 smithi196 ceph-mon[91352]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: pgmap v125: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: osdmap e86: 8 total, 8 up, 8 in 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.666 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[107701]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: pgmap v125: 89 pgs: 16 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: osdmap e86: 8 total, 8 up, 8 in 2024-04-05T02:59:42.667 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.668 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.668 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.668 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.668 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.668 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:42.668 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:42 smithi150 ceph-mon[103445]: failed to encode map e86 with expected crc 2024-04-05T02:59:43.395 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: failed to encode map e87 with expected crc 2024-04-05T02:59:43.395 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: failed to encode map e87 with expected crc 2024-04-05T02:59:43.395 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T02:59:43.395 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:43.395 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: failed to encode map e87 with expected crc 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: failed to encode map e87 with expected crc 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:59:43.396 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:43 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: failed to encode map e87 with expected crc 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: failed to encode map e87 with expected crc 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: osdmap e87: 8 total, 8 up, 8 in 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T02:59:43.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:43 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:44.593 INFO:teuthology.orchestra.run.smithi150.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[103445]: Reconfiguring osd.1 (monmap changed)... 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.1 on smithi150 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[103445]: pgmap v128: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/759888754' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[107701]: Reconfiguring osd.1 (monmap changed)... 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.1 on smithi150 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[107701]: pgmap v128: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:44.594 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/759888754' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:44.595 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:44 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:44.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:44 smithi196 ceph-mon[91352]: Reconfiguring osd.1 (monmap changed)... 2024-04-05T02:59:44.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:44 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.1 on smithi150 2024-04-05T02:59:44.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:44 smithi196 ceph-mon[91352]: pgmap v128: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:44.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:44 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/759888754' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:44.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:44 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T02:59:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: failed to encode map e88 with expected crc 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: failed to encode map e88 with expected crc 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: Reconfiguring osd.2 (monmap changed)... 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.2 on smithi150 2024-04-05T02:59:45.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:45 smithi196 ceph-mon[91352]: pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:45.951 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:59:45.951 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: failed to encode map e88 with expected crc 2024-04-05T02:59:45.951 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: failed to encode map e88 with expected crc 2024-04-05T02:59:45.951 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T02:59:45.951 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: Reconfiguring osd.2 (monmap changed)... 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.2 on smithi150 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[107701]: pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: failed to encode map e88 with expected crc 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: failed to encode map e88 with expected crc 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: osdmap e88: 8 total, 8 up, 8 in 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:45.952 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:45.953 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: Reconfiguring osd.2 (monmap changed)... 2024-04-05T02:59:45.953 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T02:59:45.953 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:45.953 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.2 on smithi150 2024-04-05T02:59:45.953 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:45 smithi150 ceph-mon[103445]: pgmap v130: 89 pgs: 89 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T02:59:46.660 DEBUG:teuthology.orchestra.run.smithi150:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-04-05T02:59:46.663 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-05T02:59:46.664 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-05T02:59:46.711 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:hints = 1 2024-04-05T02:59:46.711 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-05T02:59:46.711 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object prefix: benchmark_data_smithi150_114940 2024-04-05T02:59:46.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T02:59:46.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 0 0 0 0 0 0 - 0 2024-04-05T02:59:46.835 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:46 smithi150 ceph-mon[107701]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:59:46.835 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:46 smithi150 ceph-mon[103445]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:59:46.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:46 smithi196 ceph-mon[91352]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T02:59:47.711 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 1 16 3548 3532 220.737 220.75 0.00480097 0.00451471 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: failed to encode map e88 with expected crc 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: pgmap v131: 89 pgs: 89 active+clean; 9.2 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: Reconfiguring osd.3 (monmap changed)... 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:47.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:47 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.3 on smithi150 2024-04-05T02:59:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: failed to encode map e88 with expected crc 2024-04-05T02:59:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: pgmap v131: 89 pgs: 89 active+clean; 9.2 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: Reconfiguring osd.3 (monmap changed)... 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.3 on smithi150 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: failed to encode map e88 with expected crc 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: pgmap v131: 89 pgs: 89 active+clean; 9.2 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s 2024-04-05T02:59:47.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: Reconfiguring osd.3 (monmap changed)... 2024-04-05T02:59:47.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T02:59:47.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:47.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:47 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.3 on smithi150 2024-04-05T02:59:48.711 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 2 15 6843 6828 213.359 206 0.00365007 0.00467615 2024-04-05T02:59:49.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 3 16 10035 10019 208.713 199.438 0.00669109 0.00478313 2024-04-05T02:59:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: pgmap v132: 89 pgs: 89 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s wr, 85 op/s 2024-04-05T02:59:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:50.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:50.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: Reconfiguring mds.foo.smithi150.jnrwgs (monmap changed)... 2024-04-05T02:59:50.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:59:50.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:59:50.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:50.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:50 smithi196 ceph-mon[91352]: Reconfiguring daemon mds.foo.smithi150.jnrwgs on smithi150 2024-04-05T02:59:50.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 4 16 12944 12928 201.984 181.812 0.00665364 0.00494343 2024-04-05T02:59:50.717 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: pgmap v132: 89 pgs: 89 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s wr, 85 op/s 2024-04-05T02:59:50.717 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:50.717 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:50.717 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: Reconfiguring mds.foo.smithi150.jnrwgs (monmap changed)... 2024-04-05T02:59:50.717 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:59:50.717 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[103445]: Reconfiguring daemon mds.foo.smithi150.jnrwgs on smithi150 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: pgmap v132: 89 pgs: 89 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s wr, 85 op/s 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: Reconfiguring mds.foo.smithi150.jnrwgs (monmap changed)... 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi150.jnrwgs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:50.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:50 smithi150 ceph-mon[107701]: Reconfiguring daemon mds.foo.smithi150.jnrwgs on smithi150 2024-04-05T02:59:51.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 5 16 15899 15883 198.521 184.688 0.00716218 0.00502996 2024-04-05T02:59:51.726 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 02:59:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:02:59:51] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2024-04-05T02:59:51.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:51 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:51.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:51 smithi150 ceph-mon[103445]: pgmap v133: 89 pgs: 89 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 75 op/s 2024-04-05T02:59:51.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:51 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:51 smithi150 ceph-mon[107701]: pgmap v133: 89 pgs: 89 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 75 op/s 2024-04-05T02:59:52.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:51 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:52.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:51 smithi196 ceph-mon[91352]: pgmap v133: 89 pgs: 89 active+clean; 42 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s wr, 75 op/s 2024-04-05T02:59:52.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 6 16 18525 18509 192.786 164.125 0.00711202 0.00517916 2024-04-05T02:59:53.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:53.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:53.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:59:53.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:53.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:53.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:53.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:53 smithi196 ceph-mon[91352]: Reconfiguring daemon mon.b on smithi196 2024-04-05T02:59:53.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:53.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[103445]: Reconfiguring daemon mon.b on smithi196 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: Reconfiguring mon.b (monmap changed)... 2024-04-05T02:59:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-05T02:59:53.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-05T02:59:53.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:53.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:53 smithi150 ceph-mon[107701]: Reconfiguring daemon mon.b on smithi196 2024-04-05T02:59:53.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 7 16 21235 21219 189.439 169.375 0.00180116 0.00526464 2024-04-05T02:59:54.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:54 smithi150 ceph-mon[103445]: pgmap v134: 89 pgs: 89 active+clean; 1017 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-04-05T02:59:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:54 smithi150 ceph-mon[107701]: pgmap v134: 89 pgs: 89 active+clean; 1017 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-04-05T02:59:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:54 smithi196 ceph-mon[91352]: pgmap v134: 89 pgs: 89 active+clean; 1017 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 102 MiB/s wr, 1.62k op/s 2024-04-05T02:59:54.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 8 15 23791 23776 185.734 159.812 0.00664366 0.00537815 2024-04-05T02:59:55.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 9 16 26235 26219 182.06 152.688 0.00498693 0.00548639 2024-04-05T02:59:56.096 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:56.096 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:56.096 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: Reconfiguring mgr.x (monmap changed)... 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: Reconfiguring daemon mgr.x on smithi196 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: pgmap v135: 89 pgs: 89 active+clean; 1017 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-05T02:59:56.097 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: Reconfiguring mgr.x (monmap changed)... 2024-04-05T02:59:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: Reconfiguring daemon mgr.x on smithi196 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: pgmap v135: 89 pgs: 89 active+clean; 1017 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: Reconfiguring mgr.x (monmap changed)... 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-05T02:59:56.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:56.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: Reconfiguring daemon mgr.x on smithi196 2024-04-05T02:59:56.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: pgmap v135: 89 pgs: 89 active+clean; 1017 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-04-05T02:59:56.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T02:59:56.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 10 15 28603 28588 178.659 148.062 0.00355791 0.00559091 2024-04-05T02:59:57.712 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 11 16 30616 30600 173.848 125.75 0.00534625 0.00574696 2024-04-05T02:59:58.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: pgmap v136: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-04-05T02:59:58.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:58.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:58.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: Reconfiguring osd.4 (monmap changed)... 2024-04-05T02:59:58.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:59:58.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:58.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 02:59:58 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.4 on smithi196 2024-04-05T02:59:58.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 12 16 32802 32786 170.745 136.625 0.00674909 0.00585088 2024-04-05T02:59:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: pgmap v136: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-04-05T02:59:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: Reconfiguring osd.4 (monmap changed)... 2024-04-05T02:59:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.4 on smithi196 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: pgmap v136: 89 pgs: 89 active+clean; 1.5 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: Reconfiguring osd.4 (monmap changed)... 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T02:59:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 02:59:58 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.4 on smithi196 2024-04-05T02:59:59.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 13 16 34601 34585 166.259 112.438 0.00950728 0.00600957 2024-04-05T03:00:00.454 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:00 smithi196 ceph-mon[91352]: pgmap v137: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-04-05T03:00:00.454 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:00 smithi196 ceph-mon[91352]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:00:00.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 14 16 36216 36200 161.592 100.938 0.00806297 0.00618199 2024-04-05T03:00:00.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:00 smithi150 ceph-mon[107701]: pgmap v137: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-04-05T03:00:00.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:00 smithi150 ceph-mon[107701]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:00:00.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:00 smithi150 ceph-mon[103445]: pgmap v137: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-04-05T03:00:00.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:00 smithi150 ceph-mon[103445]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:00:01.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 15 16 38219 38203 159.165 125.188 0.0106203 0.00627746 2024-04-05T03:00:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:00:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:00:01] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2024-04-05T03:00:02.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:02 smithi196 ceph-mon[91352]: pgmap v138: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-04-05T03:00:02.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:02 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:02.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:02 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:02.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:02 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:00:02.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:02 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:02.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 16 16 40112 40096 156.611 118.312 0.0116745 0.00638056 2024-04-05T03:00:02.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[103445]: pgmap v138: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-04-05T03:00:02.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:02.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:02.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:00:02.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:02.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[107701]: pgmap v138: 89 pgs: 89 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2024-04-05T03:00:02.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:02.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:02.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-05T03:00:02.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:02 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:03.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:03 smithi196 ceph-mon[91352]: Reconfiguring osd.5 (monmap changed)... 2024-04-05T03:00:03.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:03 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.5 on smithi196 2024-04-05T03:00:03.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 17 16 42038 42022 154.478 120.375 0.00691056 0.0064681 2024-04-05T03:00:03.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:03 smithi150 ceph-mon[103445]: Reconfiguring osd.5 (monmap changed)... 2024-04-05T03:00:03.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:03 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.5 on smithi196 2024-04-05T03:00:03.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:03 smithi150 ceph-mon[107701]: Reconfiguring osd.5 (monmap changed)... 2024-04-05T03:00:03.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:03 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.5 on smithi196 2024-04-05T03:00:04.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:04 smithi196 ceph-mon[91352]: pgmap v139: 89 pgs: 89 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 197 MiB/s wr, 3.15k op/s 2024-04-05T03:00:04.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:04.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 18 16 43900 43884 152.361 116.375 0.01466 0.00655864 2024-04-05T03:00:04.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:04 smithi150 ceph-mon[103445]: pgmap v139: 89 pgs: 89 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 197 MiB/s wr, 3.15k op/s 2024-04-05T03:00:04.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:04 smithi150 ceph-mon[107701]: pgmap v139: 89 pgs: 89 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 197 MiB/s wr, 3.15k op/s 2024-04-05T03:00:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:05 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:05 smithi196 ceph-mon[91352]: Reconfiguring osd.6 (monmap changed)... 2024-04-05T03:00:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:05 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:00:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:05 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:05 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.6 on smithi196 2024-04-05T03:00:05.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 19 16 45692 45676 150.236 112 0.00488671 0.00665125 2024-04-05T03:00:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[107701]: Reconfiguring osd.6 (monmap changed)... 2024-04-05T03:00:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:00:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.6 on smithi196 2024-04-05T03:00:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[103445]: Reconfiguring osd.6 (monmap changed)... 2024-04-05T03:00:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-05T03:00:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:05.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:05 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.6 on smithi196 2024-04-05T03:00:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:06 smithi196 ceph-mon[91352]: pgmap v140: 89 pgs: 89 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-04-05T03:00:06.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:00:06.714750+0000 min lat: 0.00108641 max lat: 0.14599 avg lat: 0.0067371 2024-04-05T03:00:06.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:00:06.714 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 20 16 47486 47470 148.33 112.125 0.0157918 0.0067371 2024-04-05T03:00:06.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:06 smithi150 ceph-mon[103445]: pgmap v140: 89 pgs: 89 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-04-05T03:00:06.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:06 smithi150 ceph-mon[107701]: pgmap v140: 89 pgs: 89 active+clean; 2.4 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 116 MiB/s wr, 1.85k op/s 2024-04-05T03:00:07.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 21 16 49173 49157 146.287 105.438 0.00609001 0.00683066 2024-04-05T03:00:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: pgmap v141: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-05T03:00:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: Reconfiguring osd.7 (monmap changed)... 2024-04-05T03:00:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:00:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:08.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:08 smithi196 ceph-mon[91352]: Reconfiguring daemon osd.7 on smithi196 2024-04-05T03:00:08.714 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 22 16 50910 50894 144.572 108.562 0.0139137 0.0069115 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: pgmap v141: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: Reconfiguring osd.7 (monmap changed)... 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[103445]: Reconfiguring daemon osd.7 on smithi196 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: pgmap v141: 89 pgs: 89 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 149 MiB/s wr, 2.38k op/s 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: Reconfiguring osd.7 (monmap changed)... 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:08.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:08 smithi150 ceph-mon[107701]: Reconfiguring daemon osd.7 on smithi196 2024-04-05T03:00:09.713 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 23 15 52549 52534 142.742 102.5 0.00484913 0.00700067 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: pgmap v142: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: Reconfiguring mds.foo.smithi196.eohomm (monmap changed)... 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:10.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:10 smithi196 ceph-mon[91352]: Reconfiguring daemon mds.foo.smithi196.eohomm on smithi196 2024-04-05T03:00:10.714 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 24 16 54180 54164 141.039 101.875 0.0105816 0.00708439 2024-04-05T03:00:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: pgmap v142: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-04-05T03:00:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: Reconfiguring mds.foo.smithi196.eohomm (monmap changed)... 2024-04-05T03:00:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[107701]: Reconfiguring daemon mds.foo.smithi196.eohomm on smithi196 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: pgmap v142: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: Reconfiguring mds.foo.smithi196.eohomm (monmap changed)... 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi196.eohomm", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:10 smithi150 ceph-mon[103445]: Reconfiguring daemon mds.foo.smithi196.eohomm on smithi196 2024-04-05T03:00:11.714 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 25 16 55691 55675 139.174 94.4375 0.0139165 0.00717955 2024-04-05T03:00:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:11.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:00:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:00:11] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.43.0" 2024-04-05T03:00:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:12.715 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 26 16 56840 56824 136.58 71.8125 0.00879101 0.00731606 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[103445]: pgmap v143: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[107701]: pgmap v143: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:12.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:12 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:12.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:12 smithi196 ceph-mon[91352]: pgmap v143: 89 pgs: 89 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s 2024-04-05T03:00:12.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:12 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:12.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:12 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:12.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:12 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:13.714 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 27 16 58082 58066 134.396 77.625 0.0184373 0.0074352 2024-04-05T03:00:14.716 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 28 16 59463 59447 132.673 86.3125 0.0139503 0.00753213 2024-04-05T03:00:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:14 smithi196 ceph-mon[91352]: pgmap v144: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 140 MiB/s wr, 2.25k op/s 2024-04-05T03:00:14.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:14 smithi150 ceph-mon[107701]: pgmap v144: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 140 MiB/s wr, 2.25k op/s 2024-04-05T03:00:14.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:14 smithi150 ceph-mon[103445]: pgmap v144: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 140 MiB/s wr, 2.25k op/s 2024-04-05T03:00:15.716 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 29 16 60862 60846 131.111 87.4375 0.0103483 0.00762265 2024-04-05T03:00:16.716 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 30 16 62243 62227 129.617 86.3125 0.00661559 0.00770906 2024-04-05T03:00:16.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:16 smithi150 ceph-mon[103445]: pgmap v145: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:16.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:16 smithi150 ceph-mon[107701]: pgmap v145: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:16.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:16 smithi196 ceph-mon[91352]: pgmap v145: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:17.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 31 16 63649 63633 128.27 87.875 0.0102198 0.00779131 2024-04-05T03:00:18.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 32 16 64971 64955 126.844 82.625 0.00364016 0.00787728 2024-04-05T03:00:18.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:18 smithi196 ceph-mon[91352]: pgmap v146: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-04-05T03:00:18.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:18 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:18 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:18.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:18 smithi150 ceph-mon[107701]: pgmap v146: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-04-05T03:00:18.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:18 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:18.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:18 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:18 smithi150 ceph-mon[103445]: pgmap v146: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2024-04-05T03:00:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:18 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:18 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:19.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 33 16 66261 66245 125.443 80.625 0.0126215 0.00796688 2024-04-05T03:00:19.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:19.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:19.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:19 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:19.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:19.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:19.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:19 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:19.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:19 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:20.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 34 16 67657 67641 124.32 87.25 0.0162109 0.00803857 2024-04-05T03:00:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:20 smithi196 ceph-mon[91352]: pgmap v147: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-05T03:00:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:20 smithi196 ceph-mon[91352]: pgmap v148: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:20.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:20 smithi150 ceph-mon[103445]: pgmap v147: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-05T03:00:20.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:20 smithi150 ceph-mon[103445]: pgmap v148: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:20.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:20 smithi150 ceph-mon[107701]: pgmap v147: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-04-05T03:00:20.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:20 smithi150 ceph-mon[107701]: pgmap v148: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:21.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 35 16 69054 69038 123.262 87.3125 0.00587397 0.00810746 2024-04-05T03:00:21.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:00:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:00:21] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.43.0" 2024-04-05T03:00:22.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 36 15 70396 70381 122.17 83.9375 0.0130393 0.00817875 2024-04-05T03:00:22.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:22 smithi196 ceph-mon[91352]: pgmap v149: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:22 smithi150 ceph-mon[103445]: pgmap v149: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:22.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:22 smithi150 ceph-mon[107701]: pgmap v149: 89 pgs: 89 active+clean; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-04-05T03:00:23.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 37 16 71718 71702 121.099 82.5625 0.0100465 0.00825271 2024-04-05T03:00:24.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 38 16 73053 73037 120.108 83.4375 0.0118852 0.008321 2024-04-05T03:00:24.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:24 smithi196 ceph-mon[91352]: pgmap v150: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:24 smithi150 ceph-mon[103445]: pgmap v150: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:24 smithi150 ceph-mon[107701]: pgmap v150: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:25.717 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 39 16 74345 74329 119.098 80.75 0.0176576 0.00839044 2024-04-05T03:00:26.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:00:26.719090+0000 min lat: 0.00108641 max lat: 0.14599 avg lat: 0.00845889 2024-04-05T03:00:26.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:00:26.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 40 16 75642 75626 118.147 81.0625 0.00512707 0.00845889 2024-04-05T03:00:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:26 smithi196 ceph-mon[91352]: pgmap v151: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:26 smithi150 ceph-mon[103445]: pgmap v151: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:26.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:26 smithi150 ceph-mon[107701]: pgmap v151: 89 pgs: 89 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:26.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:27.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 41 16 76844 76828 117.098 75.125 0.0687812 0.00852747 2024-04-05T03:00:28.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 42 16 78129 78113 116.222 80.3125 0.00676941 0.00859934 2024-04-05T03:00:28.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:28 smithi196 ceph-mon[91352]: pgmap v152: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-05T03:00:28.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:28 smithi150 ceph-mon[107701]: pgmap v152: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-05T03:00:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:28 smithi150 ceph-mon[103445]: pgmap v152: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-04-05T03:00:29.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 43 16 79376 79360 115.332 77.9375 0.00501528 0.00866498 2024-04-05T03:00:29.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:29 smithi196 ceph-mon[91352]: pgmap v153: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:29.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:29 smithi150 ceph-mon[107701]: pgmap v153: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:29.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:29 smithi150 ceph-mon[103445]: pgmap v153: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s 2024-04-05T03:00:30.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 44 16 80613 80597 114.467 77.3125 0.0214915 0.00873076 2024-04-05T03:00:31.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 45 16 81830 81814 113.614 76.0625 0.0091762 0.00879651 2024-04-05T03:00:31.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:00:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:00:31] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-05T03:00:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:32 smithi196 ceph-mon[91352]: pgmap v154: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-05T03:00:32.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 46 16 83027 83011 112.77 74.8125 0.00693908 0.0088626 2024-04-05T03:00:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:32 smithi150 ceph-mon[103445]: pgmap v154: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-05T03:00:32.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:32 smithi150 ceph-mon[107701]: pgmap v154: 89 pgs: 89 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-04-05T03:00:33.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 47 16 84219 84203 111.956 74.5 0.0125987 0.00892708 2024-04-05T03:00:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:34 smithi196 ceph-mon[91352]: pgmap v155: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-05T03:00:34.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 48 16 85309 85293 111.043 68.125 0.0116127 0.00900058 2024-04-05T03:00:34.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:34 smithi150 ceph-mon[103445]: pgmap v155: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-05T03:00:34.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:34 smithi150 ceph-mon[107701]: pgmap v155: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-04-05T03:00:35.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 49 16 86387 86371 110.151 67.375 0.0139373 0.00907252 2024-04-05T03:00:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:36 smithi196 ceph-mon[91352]: pgmap v156: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-04-05T03:00:36.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 50 16 87481 87465 109.316 68.375 0.0038128 0.00914092 2024-04-05T03:00:36.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:36 smithi150 ceph-mon[103445]: pgmap v156: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-04-05T03:00:36.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:36 smithi150 ceph-mon[107701]: pgmap v156: 89 pgs: 89 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-04-05T03:00:37.718 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 51 16 88550 88534 108.482 66.8125 0.00703068 0.00921326 2024-04-05T03:00:37.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade resume' 2024-04-05T03:00:38.451 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:00:38.595 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:38 smithi150 ceph-mon[107701]: pgmap v157: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-05T03:00:38.596 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:38 smithi150 ceph-mon[103445]: pgmap v157: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-05T03:00:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:38 smithi196 ceph-mon[91352]: pgmap v157: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-04-05T03:00:38.719 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 52 16 89573 89557 107.626 63.9375 0.00877971 0.00928605 2024-04-05T03:00:39.719 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 53 16 90692 90676 106.915 69.9375 0.00825774 0.00934753 2024-04-05T03:00:40.336 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:40 smithi150 ceph-mon[103445]: pgmap v158: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-05T03:00:40.337 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:40 smithi150 ceph-mon[107701]: pgmap v158: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-05T03:00:40.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:40 smithi196 ceph-mon[91352]: pgmap v158: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-04-05T03:00:40.719 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 54 16 91756 91740 106.166 66.5 0.00775355 0.00941387 2024-04-05T03:00:41.574 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:41.575 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:00:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:00:41] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-05T03:00:41.575 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:41.575 INFO:teuthology.orchestra.run.smithi150.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:00:41.594 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:41.719 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 55 16 92679 92663 105.285 57.6875 0.0249678 0.00949304 2024-04-05T03:00:42.342 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'echo wait for half of osds to upgrade' 2024-04-05T03:00:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:42 smithi196 ceph-mon[91352]: pgmap v159: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-04-05T03:00:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:42 smithi196 ceph-mon[91352]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:42 smithi196 ceph-mon[91352]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:00:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:42 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:42.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:42 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:42.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 56 16 93561 93545 104.385 55.125 0.0218879 0.00957387 2024-04-05T03:00:42.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[103445]: pgmap v159: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-04-05T03:00:42.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[103445]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:42.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[103445]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:00:42.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:42.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:42.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[107701]: pgmap v159: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s 2024-04-05T03:00:42.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[107701]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:00:42.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[107701]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:00:42.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:42.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:42 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:42.813 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:00:43.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 57 15 94496 94481 103.58 58.5 0.0119904 0.00964829 2024-04-05T03:00:44.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:44 smithi150 ceph-mon[107701]: pgmap v160: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-04-05T03:00:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:44 smithi150 ceph-mon[103445]: pgmap v160: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-04-05T03:00:44.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:44 smithi196 ceph-mon[91352]: pgmap v160: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-04-05T03:00:44.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 58 16 95545 95529 102.924 65.5 0.0154206 0.00971021 2024-04-05T03:00:44.803 INFO:teuthology.orchestra.run.smithi150.stdout:wait for half of osds to upgrade 2024-04-05T03:00:45.648 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-05T03:00:45.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 59 16 96598 96582 102.295 65.8125 0.0170285 0.0097706 2024-04-05T03:00:46.100 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:00:46.519 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:46 smithi150 ceph-mon[107701]: pgmap v161: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2024-04-05T03:00:46.519 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:46 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:46.519 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:46 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:46.520 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:46 smithi150 ceph-mon[103445]: pgmap v161: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2024-04-05T03:00:46.520 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:46 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:46.520 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:46 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:46.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:46 smithi196 ceph-mon[91352]: pgmap v161: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 898 op/s 2024-04-05T03:00:46.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:46 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:46.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:46 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:46.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:00:46.722621+0000 min lat: 0.00108641 max lat: 0.14599 avg lat: 0.00983828 2024-04-05T03:00:46.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:00:46.721 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 60 16 97554 97538 101.586 59.75 0.0170949 0.00983828 2024-04-05T03:00:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:47 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:47 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:47 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:47 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:47.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:47.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:47 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:47.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:47 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:00:47.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:47.723 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 61 16 98529 98513 100.915 60.9375 0.0116136 0.00990299 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: pgmap v162: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: pgmap v163: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: Upgrade: Setting container_image for all mon 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:00:48.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: Upgrade: Setting container_image for all crash 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:00:48.536 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[107701]: Upgrade: osd.0 is safe to restart 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: pgmap v162: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: pgmap v163: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: Upgrade: Setting container_image for all mon 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:00:48.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: Upgrade: Setting container_image for all crash 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:00:48.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:48 smithi150 ceph-mon[103445]: Upgrade: osd.0 is safe to restart 2024-04-05T03:00:48.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: pgmap v162: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-04-05T03:00:48.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: pgmap v163: 89 pgs: 89 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-04-05T03:00:48.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:00:48.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: Upgrade: Setting container_image for all mon 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: Upgrade: Setting container_image for all crash 2024-04-05T03:00:48.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:48.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:00:48.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-05T03:00:48.639 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:48 smithi196 ceph-mon[91352]: Upgrade: osd.0 is safe to restart 2024-04-05T03:00:48.723 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 62 16 99400 99384 100.166 54.4375 0.015489 0.00997758 2024-04-05T03:00:49.724 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 63 16 100371 100355 99.5391 60.6875 0.0124287 0.0100409 2024-04-05T03:00:49.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:49 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1202974246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:49.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:49 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1202974246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:49.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:49 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1202974246' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:50.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 64 15 101278 101263 98.8692 56.75 0.0146314 0.0101093 2024-04-05T03:00:50.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[103445]: pgmap v164: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T03:00:50.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:50.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:00:50.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:50.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[107701]: pgmap v164: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T03:00:50.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:50.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:00:50.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:50 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:50.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:50 smithi196 ceph-mon[91352]: pgmap v164: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T03:00:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:50 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:50 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-05T03:00:50.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:50 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:00:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:51 smithi150 ceph-mon[103445]: Upgrade: Updating osd.0 2024-04-05T03:00:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:51 smithi150 ceph-mon[103445]: Deploying daemon osd.0 on smithi150 2024-04-05T03:00:51.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:51 smithi150 ceph-mon[107701]: Upgrade: Updating osd.0 2024-04-05T03:00:51.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:51 smithi150 ceph-mon[107701]: Deploying daemon osd.0 on smithi150 2024-04-05T03:00:51.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 65 16 102197 102181 98.2308 57.375 0.0144541 0.010175 2024-04-05T03:00:51.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:51 smithi196 ceph-mon[91352]: Upgrade: Updating osd.0 2024-04-05T03:00:51.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:51 smithi196 ceph-mon[91352]: Deploying daemon osd.0 on smithi150 2024-04-05T03:00:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:00:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:00:51] "GET /metrics HTTP/1.1" 200 36356 "" "Prometheus/2.43.0" 2024-04-05T03:00:52.623 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:52 smithi150 ceph-mon[107701]: pgmap v165: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T03:00:52.623 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3254310479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:52.623 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:52 smithi150 ceph-mon[103445]: pgmap v165: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T03:00:52.623 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3254310479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:52.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 66 16 103144 103128 97.6393 59.1875 0.00845348 0.010236 2024-04-05T03:00:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:52 smithi196 ceph-mon[91352]: pgmap v165: 89 pgs: 89 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-04-05T03:00:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3254310479' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:53.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 67 16 104033 104017 97.0112 55.5625 0.0179301 0.0103026 2024-04-05T03:00:54.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 68 16 104887 104871 96.3695 53.375 0.00717854 0.0103718 2024-04-05T03:00:54.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-mon[103445]: pgmap v166: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-04-05T03:00:54.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/191247870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-mon[107701]: pgmap v166: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-04-05T03:00:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/191247870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:54.728 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:54 smithi150 systemd[1]: Stopping Ceph osd.0 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:00:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:54 smithi196 ceph-mon[91352]: pgmap v166: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-04-05T03:00:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:54 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/191247870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:55.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[46261]: 2024-04-05T03:00:54.774+0000 7fef13349700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:00:55.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[46261]: 2024-04-05T03:00:54.774+0000 7fef13349700 -1 osd.0 88 *** Got signal Terminated *** 2024-04-05T03:00:55.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[46261]: 2024-04-05T03:00:54.774+0000 7fef13349700 -1 osd.0 88 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:00:55.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 69 16 105604 105588 95.6223 44.8125 0.00376112 0.0104098 2024-04-05T03:00:55.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:55 smithi150 ceph-mon[103445]: osd.0 marked itself down and dead 2024-04-05T03:00:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:55 smithi150 ceph-mon[107701]: osd.0 marked itself down and dead 2024-04-05T03:00:55.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:55 smithi196 ceph-mon[91352]: osd.0 marked itself down and dead 2024-04-05T03:00:56.293 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:56 smithi150 podman[116949]: 2024-04-05 03:00:56.020019562 +0000 UTC m=+1.406122568 container died a6b8a01405406545019373fad78c5df38e705a00c018602e518fe99a87d8bf2c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-05T03:00:56.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 70 16 105930 105914 94.5474 20.375 0.00570932 0.0105716 2024-04-05T03:00:56.727 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:56 smithi150 podman[116949]: 2024-04-05 03:00:56.478882703 +0000 UTC m=+1.864985696 container cleanup a6b8a01405406545019373fad78c5df38e705a00c018602e518fe99a87d8bf2c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=quincy-6fda41f, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:00:56.727 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:56 smithi150 bash[116949]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: pgmap v167: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: failed to encode map e89 with expected crc 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: failed to encode map e89 with expected crc 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: osdmap e89: 8 total, 7 up, 8 in 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:56.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/211901652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: pgmap v167: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: failed to encode map e89 with expected crc 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: failed to encode map e89 with expected crc 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: osdmap e89: 8 total, 7 up, 8 in 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:56.731 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:56 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/211901652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: pgmap v167: 89 pgs: 89 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: failed to encode map e89 with expected crc 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: failed to encode map e89 with expected crc 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: osdmap e89: 8 total, 7 up, 8 in 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]: dispatch 2024-04-05T03:00:56.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]: dispatch 2024-04-05T03:00:56.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-05T03:00:56.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:00:56.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:00:56.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:56 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/211901652' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:57.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:56 smithi150 podman[116969]: 2024-04-05 03:00:56.837397853 +0000 UTC m=+0.809071320 container remove a6b8a01405406545019373fad78c5df38e705a00c018602e518fe99a87d8bf2c (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=quincy-6fda41f, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:00:57.666 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:57 smithi150 podman[117074]: 2024-04-05 03:00:57.282132656 +0000 UTC m=+0.031937201 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:00:57.666 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:57 smithi150 podman[117074]: 2024-04-05 03:00:57.413517728 +0000 UTC m=+0.163322244 container create 514e5aca65d463070adfcbfe2916708986ae9c008dc386555b835e79e8e922ce (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-05T03:00:57.667 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:57 smithi150 podman[117074]: 2024-04-05 03:00:57.666935144 +0000 UTC m=+0.416739668 container init 514e5aca65d463070adfcbfe2916708986ae9c008dc386555b835e79e8e922ce (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]': finished 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]': finished 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]': finished 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-05T03:00:57.667 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]': finished 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]': finished 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: osdmap e90: 8 total, 7 up, 8 in 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.668 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: osdmap e91: 8 total, 7 up, 8 in 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]': finished 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]': finished 2024-04-05T03:00:57.669 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: osdmap e90: 8 total, 7 up, 8 in 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.670 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: osdmap e91: 8 total, 7 up, 8 in 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.671 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:57 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 71 16 107117 107101 94.2606 74.1875 0.0148806 0.0106038 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 5]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1", "id": [4, 0]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.c", "id": [3, 5]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [4, 0]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [3, 5]}]': finished 2024-04-05T03:00:57.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5, 3, 0]}]': finished 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: osdmap e90: 8 total, 7 up, 8 in 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e90 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: osdmap e91: 8 total, 7 up, 8 in 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:57 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:57.977 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:57 smithi150 podman[117074]: 2024-04-05 03:00:57.67568812 +0000 UTC m=+0.425492636 container start 514e5aca65d463070adfcbfe2916708986ae9c008dc386555b835e79e8e922ce (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, io.buildah.version=1.33.5, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-05T03:00:57.978 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:57 smithi150 podman[117074]: 2024-04-05 03:00:57.775216806 +0000 UTC m=+0.525021334 container attach 514e5aca65d463070adfcbfe2916708986ae9c008dc386555b835e79e8e922ce (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-04-05T03:00:58.428 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:58 smithi150 podman[117074]: 2024-04-05 03:00:58.162184665 +0000 UTC m=+0.911989186 container died 514e5aca65d463070adfcbfe2916708986ae9c008dc386555b835e79e8e922ce (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403) 2024-04-05T03:00:58.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 72 16 108317 108301 93.993 75 0.0118804 0.0106342 2024-04-05T03:00:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: pgmap v171: 89 pgs: 4 peering, 11 active+undersized+degraded, 12 active+undersized, 59 active+clean, 3 stale+active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 39637/314226 objects degraded (12.614%) 2024-04-05T03:00:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: Health check failed: Degraded data redundancy: 39637/314226 objects degraded (12.614%), 11 pgs degraded (PG_DEGRADED) 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: osdmap e92: 8 total, 7 up, 8 in 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: failed to encode map e92 with expected crc 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[103445]: failed to encode map e92 with expected crc 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: pgmap v171: 89 pgs: 4 peering, 11 active+undersized+degraded, 12 active+undersized, 59 active+clean, 3 stale+active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 39637/314226 objects degraded (12.614%) 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: Health check failed: Degraded data redundancy: 39637/314226 objects degraded (12.614%), 11 pgs degraded (PG_DEGRADED) 2024-04-05T03:00:58.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: osdmap e92: 8 total, 7 up, 8 in 2024-04-05T03:00:58.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: failed to encode map e92 with expected crc 2024-04-05T03:00:58.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:58 smithi150 ceph-mon[107701]: failed to encode map e92 with expected crc 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: failed to encode map e91 with expected crc 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: pgmap v171: 89 pgs: 4 peering, 11 active+undersized+degraded, 12 active+undersized, 59 active+clean, 3 stale+active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 39637/314226 objects degraded (12.614%) 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: Health check failed: Degraded data redundancy: 39637/314226 objects degraded (12.614%), 11 pgs degraded (PG_DEGRADED) 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: osdmap e92: 8 total, 7 up, 8 in 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: failed to encode map e92 with expected crc 2024-04-05T03:00:58.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:58 smithi196 ceph-mon[91352]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.444 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 podman[117124]: 2024-04-05 03:00:59.072715837 +0000 UTC m=+0.903157995 container remove 514e5aca65d463070adfcbfe2916708986ae9c008dc386555b835e79e8e922ce (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:00:59.444 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.0.service: Deactivated successfully. 2024-04-05T03:00:59.444 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 systemd[1]: Stopped Ceph osd.0 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:00:59.444 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.0.service: Consumed 15min 10.371s CPU time. 2024-04-05T03:00:59.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 73 16 109456 109440 93.6805 71.1875 0.00622802 0.0106694 2024-04-05T03:00:59.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[107701]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[107701]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[107701]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/244030280' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:59.727 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 systemd[1]: Starting Ceph osd.0 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:00:59.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[103445]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[103445]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[103445]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:00:59 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/244030280' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:00:59.787 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:59 smithi196 ceph-mon[91352]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.787 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:59 smithi196 ceph-mon[91352]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:59 smithi196 ceph-mon[91352]: failed to encode map e92 with expected crc 2024-04-05T03:00:59.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:00:59 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/244030280' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:00.137 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 podman[117242]: 2024-04-05 03:00:59.686469659 +0000 UTC m=+0.033369041 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:01:00.137 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:00:59 smithi150 podman[117242]: 2024-04-05 03:00:59.841322358 +0000 UTC m=+0.188221719 container create 649c00153258807a36700b47ced1951b064f745a4c3e5fb46398be654af9ab9f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:01:00.445 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 podman[117242]: 2024-04-05 03:01:00.137922697 +0000 UTC m=+0.484822071 container init 649c00153258807a36700b47ced1951b064f745a4c3e5fb46398be654af9ab9f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, io.buildah.version=1.33.5) 2024-04-05T03:01:00.445 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 podman[117242]: 2024-04-05 03:01:00.149450575 +0000 UTC m=+0.496349940 container start 649c00153258807a36700b47ced1951b064f745a4c3e5fb46398be654af9ab9f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-0028525, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:01:00.445 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 podman[117242]: 2024-04-05 03:01:00.252785295 +0000 UTC m=+0.599684669 container attach 649c00153258807a36700b47ced1951b064f745a4c3e5fb46398be654af9ab9f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-05T03:01:00.726 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 74 16 110652 110636 93.4247 74.75 0.00592892 0.0106988 2024-04-05T03:01:00.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-mon[103445]: pgmap v173: 89 pgs: 3 remapped+peering, 4 peering, 15 active+undersized+degraded, 16 active+undersized, 51 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 52971/318552 objects degraded (16.629%) 2024-04-05T03:01:00.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-mon[107701]: pgmap v173: 89 pgs: 3 remapped+peering, 4 peering, 15 active+undersized+degraded, 16 active+undersized, 51 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 52971/318552 objects degraded (16.629%) 2024-04-05T03:01:00.796 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:00 smithi196 ceph-mon[91352]: pgmap v173: 89 pgs: 3 remapped+peering, 4 peering, 15 active+undersized+degraded, 16 active+undersized, 51 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 52971/318552 objects degraded (16.629%) 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate[117254]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-05T03:01:01.228 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-05T03:01:01.229 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-05T03:01:01.229 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 bash[117242]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-05T03:01:01.229 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:00 smithi150 podman[117242]: 2024-04-05 03:01:00.945513263 +0000 UTC m=+1.292412624 container died 649c00153258807a36700b47ced1951b064f745a4c3e5fb46398be654af9ab9f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-05T03:01:01.582 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:01:01] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.43.0" 2024-04-05T03:01:01.726 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 75 16 111765 111749 93.1064 69.5625 0.0112397 0.0107354 2024-04-05T03:01:01.912 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:01 smithi150 podman[117395]: 2024-04-05 03:01:01.583448026 +0000 UTC m=+0.630382567 container remove 649c00153258807a36700b47ced1951b064f745a4c3e5fb46398be654af9ab9f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_BRANCH=HEAD) 2024-04-05T03:01:02.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:01 smithi150 podman[117481]: 2024-04-05 03:01:01.816136915 +0000 UTC m=+0.042705448 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:01:02.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:02 smithi150 podman[117481]: 2024-04-05 03:01:02.109935024 +0000 UTC m=+0.336503546 container create 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-05T03:01:02.707 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:02 smithi150 ceph-mon[103445]: pgmap v174: 89 pgs: 3 remapped+peering, 4 peering, 15 active+undersized+degraded, 16 active+undersized, 51 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 52971/318552 objects degraded (16.629%) 2024-04-05T03:01:02.707 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:02 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/839280384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:02.707 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:02 smithi150 ceph-mon[107701]: pgmap v174: 89 pgs: 3 remapped+peering, 4 peering, 15 active+undersized+degraded, 16 active+undersized, 51 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 52971/318552 objects degraded (16.629%) 2024-04-05T03:01:02.708 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:02 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/839280384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:02.708 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:02 smithi150 podman[117481]: 2024-04-05 03:01:02.69718207 +0000 UTC m=+0.923750592 container init 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-0028525, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:01:02.708 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:02 smithi150 podman[117481]: 2024-04-05 03:01:02.707676679 +0000 UTC m=+0.934245195 container start 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=squid-0028525, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-05T03:01:02.725 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 76 16 112942 112926 92.8492 73.5625 0.012488 0.010765 2024-04-05T03:01:02.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:02 smithi196 ceph-mon[91352]: pgmap v174: 89 pgs: 3 remapped+peering, 4 peering, 15 active+undersized+degraded, 16 active+undersized, 51 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 52971/318552 objects degraded (16.629%) 2024-04-05T03:01:02.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:02 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/839280384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:02.977 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:02 smithi150 bash[117481]: 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e 2024-04-05T03:01:02.977 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:02 smithi150 systemd[1]: Started Ceph osd.0 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:01:03.477 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[117509]: 2024-04-05T03:01:03.017+0000 7f702054e7c0 -1 Falling back to public interface 2024-04-05T03:01:03.727 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 77 16 114033 114017 92.5281 68.1875 0.0141891 0.0108016 2024-04-05T03:01:04.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:04.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:04.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:04.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[107701]: pgmap v175: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 55879/336093 objects degraded (16.626%) 2024-04-05T03:01:04.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:04.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:04.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:04.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:03 smithi150 ceph-mon[103445]: pgmap v175: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 55879/336093 objects degraded (16.626%) 2024-04-05T03:01:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:03 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:03 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:03 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:03 smithi196 ceph-mon[91352]: pgmap v175: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 55879/336093 objects degraded (16.626%) 2024-04-05T03:01:04.726 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 78 16 115122 115106 92.2144 68.0625 0.0134635 0.0108391 2024-04-05T03:01:05.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:04 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 55879/336093 objects degraded (16.626%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:05.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:04 smithi150 ceph-mon[107701]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-05T03:01:05.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:04 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3974142149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:04 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 55879/336093 objects degraded (16.626%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:04 smithi150 ceph-mon[103445]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-05T03:01:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:04 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3974142149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:04 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 55879/336093 objects degraded (16.626%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:04 smithi196 ceph-mon[91352]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-04-05T03:01:05.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:04 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3974142149' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:05.727 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 79 16 116160 116144 91.8683 64.875 0.0406367 0.0108791 2024-04-05T03:01:06.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:05 smithi150 ceph-mon[107701]: pgmap v176: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 55879/336093 objects degraded (16.626%) 2024-04-05T03:01:06.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:05 smithi150 ceph-mon[103445]: pgmap v176: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 55879/336093 objects degraded (16.626%) 2024-04-05T03:01:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:05 smithi196 ceph-mon[91352]: pgmap v176: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 55879/336093 objects degraded (16.626%) 2024-04-05T03:01:06.730 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:01:06.731660+0000 min lat: 0.00108641 max lat: 1.04178 avg lat: 0.0109256 2024-04-05T03:01:06.730 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:01:06.730 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 80 16 117139 117123 91.4809 61.1875 0.0254522 0.0109256 2024-04-05T03:01:07.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:06 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1837509142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:07.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:06 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1837509142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:07.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:06 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1837509142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:07.730 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 81 16 118151 118135 91.1323 63.25 0.00657029 0.0109677 2024-04-05T03:01:08.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:07 smithi150 ceph-mon[103445]: pgmap v177: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 57885/346308 objects degraded (16.715%) 2024-04-05T03:01:08.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:07 smithi150 ceph-mon[107701]: pgmap v177: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 57885/346308 objects degraded (16.715%) 2024-04-05T03:01:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:07 smithi196 ceph-mon[91352]: pgmap v177: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 57885/346308 objects degraded (16.715%) 2024-04-05T03:01:08.730 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 82 16 119228 119212 90.8418 67.3125 0.00802194 0.011002 2024-04-05T03:01:09.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:09 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 58542/352149 objects degraded (16.624%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:09 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 58542/352149 objects degraded (16.624%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:09.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:09 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 58542/352149 objects degraded (16.624%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:09.732 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 83 16 120045 120029 90.3609 51.0625 0.0332655 0.011061 2024-04-05T03:01:10.628 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:10 smithi150 ceph-mon[107701]: pgmap v178: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 58542/352149 objects degraded (16.624%) 2024-04-05T03:01:10.629 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1554014902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:10.629 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:10 smithi150 ceph-mon[103445]: pgmap v178: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 58542/352149 objects degraded (16.624%) 2024-04-05T03:01:10.629 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1554014902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:10.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:10 smithi196 ceph-mon[91352]: pgmap v178: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 58542/352149 objects degraded (16.624%) 2024-04-05T03:01:10.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1554014902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:10.732 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 84 16 121007 120991 90.0009 60.125 0.0174439 0.0111042 2024-04-05T03:01:11.687 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:01:11] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2024-04-05T03:01:11.732 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 85 16 121733 121717 89.4759 45.375 0.0114855 0.0111701 2024-04-05T03:01:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[103445]: pgmap v179: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s; 58542/352149 objects degraded (16.624%) 2024-04-05T03:01:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:11 smithi150 ceph-mon[107701]: pgmap v179: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s; 58542/352149 objects degraded (16.624%) 2024-04-05T03:01:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:11 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:12.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:11 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:12.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:11 smithi196 ceph-mon[91352]: pgmap v179: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s; 58542/352149 objects degraded (16.624%) 2024-04-05T03:01:12.732 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 86 16 122477 122461 88.9762 46.5 0.0171476 0.0112329 2024-04-05T03:01:12.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1224835328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:12.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:12.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:12.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1224835328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:12.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:12.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:12.978 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:12 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[117509]: 2024-04-05T03:01:12.749+0000 7f702054e7c0 -1 osd.0 88 log_to_monitors true 2024-04-05T03:01:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:12 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1224835328' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:12 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:12 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:13.732 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 87 16 123522 123506 88.7042 65.3125 0.0136339 0.0112673 2024-04-05T03:01:14.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:13 smithi196 ceph-mon[91352]: from='osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:01:14.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:13 smithi196 ceph-mon[91352]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:01:14.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:13 smithi196 ceph-mon[91352]: pgmap v180: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 60782/365724 objects degraded (16.620%) 2024-04-05T03:01:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:13 smithi150 ceph-mon[103445]: from='osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:01:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:13 smithi150 ceph-mon[103445]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:01:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:13 smithi150 ceph-mon[103445]: pgmap v180: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 60782/365724 objects degraded (16.620%) 2024-04-05T03:01:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:13 smithi150 ceph-mon[107701]: from='osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:01:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:13 smithi150 ceph-mon[107701]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-05T03:01:14.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:13 smithi150 ceph-mon[107701]: pgmap v180: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 60782/365724 objects degraded (16.620%) 2024-04-05T03:01:14.727 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[117509]: 2024-04-05T03:01:14.348+0000 7f700d99e700 -1 osd.0 88 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:01:14.732 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 88 16 124180 124164 88.1636 41.125 0.0132032 0.0113373 2024-04-05T03:01:15.048 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T03:01:15.048 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.048 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: osdmap e93: 8 total, 7 up, 8 in 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='osd.0 ' entity='osd.0' 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3765260191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.049 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: osdmap e93: 8 total, 7 up, 8 in 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='osd.0 ' entity='osd.0' 2024-04-05T03:01:15.050 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:14 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3765260191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:15.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-05T03:01:15.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: osdmap e93: 8 total, 7 up, 8 in 2024-04-05T03:01:15.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:01:15.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:01:15.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: failed to encode map e93 with expected crc 2024-04-05T03:01:15.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:15.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:15.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='osd.0 ' entity='osd.0' 2024-04-05T03:01:15.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:14 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3765260191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:15.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 89 16 124534 124518 87.4216 22.125 0.0037202 0.0113519 2024-04-05T03:01:16.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:01:16.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: failed to encode map e94 with expected crc 2024-04-05T03:01:16.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: failed to encode map e94 with expected crc 2024-04-05T03:01:16.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590] boot 2024-04-05T03:01:16.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: osdmap e94: 8 total, 8 up, 8 in 2024-04-05T03:01:16.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:01:16.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:15 smithi196 ceph-mon[91352]: pgmap v183: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 60782/365724 objects degraded (16.620%) 2024-04-05T03:01:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:01:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: failed to encode map e94 with expected crc 2024-04-05T03:01:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: failed to encode map e94 with expected crc 2024-04-05T03:01:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590] boot 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: osdmap e94: 8 total, 8 up, 8 in 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[107701]: pgmap v183: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 60782/365724 objects degraded (16.620%) 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: failed to encode map e94 with expected crc 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: failed to encode map e94 with expected crc 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: osd.0 [v2:172.21.15.150:6802/4017397590,v1:172.21.15.150:6803/4017397590] boot 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: osdmap e94: 8 total, 8 up, 8 in 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-05T03:01:16.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:15 smithi150 ceph-mon[103445]: pgmap v183: 89 pgs: 6 active+clean+remapped, 15 active+undersized+degraded, 16 active+undersized, 52 active+clean; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 808 op/s; 60782/365724 objects degraded (16.620%) 2024-04-05T03:01:16.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 90 16 124534 124518 86.4504 0 - 0.0113519 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: osdmap e95: 8 total, 8 up, 8 in 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/695149929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:17.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 60782/365724 objects degraded (16.620%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: osdmap e95: 8 total, 8 up, 8 in 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/695149929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 60782/365724 objects degraded (16.620%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: osdmap e95: 8 total, 8 up, 8 in 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: failed to encode map e95 with expected crc 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/695149929' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:17.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 60782/365724 objects degraded (16.620%), 15 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total time run: 90.5149 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total writes made: 124534 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Write size: 65536 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object size: 65536 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Bandwidth (MB/sec): 85.99 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Bandwidth: 40.5757 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max bandwidth (MB/sec): 220.75 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min bandwidth (MB/sec): 0 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average IOPS: 1375 2024-04-05T03:01:17.733 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev IOPS: 649.211 2024-04-05T03:01:17.734 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max IOPS: 3532 2024-04-05T03:01:17.734 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min IOPS: 0 2024-04-05T03:01:17.734 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average Latency(s): 0.0116237 2024-04-05T03:01:17.734 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Latency(s): 0.0277333 2024-04-05T03:01:17.734 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max latency(s): 2.15091 2024-04-05T03:01:17.734 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min latency(s): 0.00108641 2024-04-05T03:01:17.740 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Cleaning up (deleting benchmark objects) 2024-04-05T03:01:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: osdmap e96: 8 total, 8 up, 8 in 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[107701]: pgmap v186: 89 pgs: 5 remapped+peering, 5 active+clean+remapped, 1 peering, 7 active+undersized+degraded, 11 active+undersized, 60 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 24 MiB/s wr, 390 op/s; 22721/370407 objects degraded (6.134%) 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: osdmap e96: 8 total, 8 up, 8 in 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-mon[103445]: pgmap v186: 89 pgs: 5 remapped+peering, 5 active+clean+remapped, 1 peering, 7 active+undersized+degraded, 11 active+undersized, 60 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 24 MiB/s wr, 390 op/s; 22721/370407 objects degraded (6.134%) 2024-04-05T03:01:18.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: osdmap e96: 8 total, 8 up, 8 in 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: failed to encode map e96 with expected crc 2024-04-05T03:01:18.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:18 smithi196 ceph-mon[91352]: pgmap v186: 89 pgs: 5 remapped+peering, 5 active+clean+remapped, 1 peering, 7 active+undersized+degraded, 11 active+undersized, 60 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 24 MiB/s wr, 390 op/s; 22721/370407 objects degraded (6.134%) 2024-04-05T03:01:19.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:18 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:01:18.971+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: pgmap v187: 89 pgs: 67 active+clean, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 1 peering, 3 active+clean+remapped, 5 remapped+peering; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 33 MiB/s wr, 524 op/s; 3572/373644 objects degraded (0.956%); 13 KiB/s, 0 objects/s recovering 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:20.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-05T03:01:20.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:19 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/602233343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:20.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: pgmap v187: 89 pgs: 67 active+clean, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 1 peering, 3 active+clean+remapped, 5 remapped+peering; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 33 MiB/s wr, 524 op/s; 3572/373644 objects degraded (0.956%); 13 KiB/s, 0 objects/s recovering 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-05T03:01:20.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/602233343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: pgmap v187: 89 pgs: 67 active+clean, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 1 peering, 3 active+clean+remapped, 5 remapped+peering; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 33 MiB/s wr, 524 op/s; 3572/373644 objects degraded (0.956%); 13 KiB/s, 0 objects/s recovering 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:01:20.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:20.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:20.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-04-05T03:01:20.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:19 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/602233343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:21.804 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:01:21] "GET /metrics HTTP/1.1" 200 36488 "" "Prometheus/2.43.0" 2024-04-05T03:01:22.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:21 smithi196 ceph-mon[91352]: pgmap v188: 89 pgs: 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 5 remapped+peering; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 29 MiB/s wr, 459 op/s; 3569/373644 objects degraded (0.955%); 11 KiB/s, 0 objects/s recovering 2024-04-05T03:01:22.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:21 smithi150 ceph-mon[107701]: pgmap v188: 89 pgs: 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 5 remapped+peering; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 29 MiB/s wr, 459 op/s; 3569/373644 objects degraded (0.955%); 11 KiB/s, 0 objects/s recovering 2024-04-05T03:01:22.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:21 smithi150 ceph-mon[103445]: pgmap v188: 89 pgs: 80 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 5 remapped+peering; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 29 MiB/s wr, 459 op/s; 3569/373644 objects degraded (0.955%); 11 KiB/s, 0 objects/s recovering 2024-04-05T03:01:23.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4294300971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 3569/373644 objects degraded (0.955%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:23.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4294300971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:23.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 3569/373644 objects degraded (0.955%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4294300971' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:23.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 3569/373644 objects degraded (0.955%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:23 smithi196 ceph-mon[91352]: pgmap v189: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 134 B/s rd, 22 MiB/s wr, 788 op/s; 9104/363588 objects degraded (2.504%); 2.3 MiB/s, 37 objects/s recovering 2024-04-05T03:01:24.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:23 smithi150 ceph-mon[103445]: pgmap v189: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 134 B/s rd, 22 MiB/s wr, 788 op/s; 9104/363588 objects degraded (2.504%); 2.3 MiB/s, 37 objects/s recovering 2024-04-05T03:01:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:23 smithi150 ceph-mon[107701]: pgmap v189: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 134 B/s rd, 22 MiB/s wr, 788 op/s; 9104/363588 objects degraded (2.504%); 2.3 MiB/s, 37 objects/s recovering 2024-04-05T03:01:25.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:24 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3482484461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:25.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:24 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3482484461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:25.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:24 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3482484461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:25 smithi150 ceph-mon[103445]: pgmap v190: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 117 B/s rd, 19 MiB/s wr, 687 op/s; 9104/363588 objects degraded (2.504%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:01:26.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:25 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:25 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:25 smithi150 ceph-mon[107701]: pgmap v190: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 117 B/s rd, 19 MiB/s wr, 687 op/s; 9104/363588 objects degraded (2.504%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:01:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:25 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:25 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:25 smithi196 ceph-mon[91352]: pgmap v190: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 117 B/s rd, 19 MiB/s wr, 687 op/s; 9104/363588 objects degraded (2.504%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:01:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:25 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:01:26.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:25 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:27 smithi150 ceph-mon[107701]: pgmap v191: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 105 B/s rd, 7.0 MiB/s wr, 609 op/s; 9104/359148 objects degraded (2.535%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:01:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9104/359148 objects degraded (2.535%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:27 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/130801813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:27 smithi150 ceph-mon[103445]: pgmap v191: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 105 B/s rd, 7.0 MiB/s wr, 609 op/s; 9104/359148 objects degraded (2.535%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:01:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9104/359148 objects degraded (2.535%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:28.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:27 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/130801813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:28.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:27 smithi196 ceph-mon[91352]: pgmap v191: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 105 B/s rd, 7.0 MiB/s wr, 609 op/s; 9104/359148 objects degraded (2.535%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:01:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9104/359148 objects degraded (2.535%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:27 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/130801813' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:30.188 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:29 smithi196 ceph-mon[91352]: pgmap v192: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88 B/s rd, 5.9 MiB/s wr, 719 op/s; 8899/351864 objects degraded (2.529%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:01:30.188 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/242513877' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:30.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:29 smithi150 ceph-mon[107701]: pgmap v192: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88 B/s rd, 5.9 MiB/s wr, 719 op/s; 8899/351864 objects degraded (2.529%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:01:30.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/242513877' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:29 smithi150 ceph-mon[103445]: pgmap v192: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 88 B/s rd, 5.9 MiB/s wr, 719 op/s; 8899/351864 objects degraded (2.529%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:01:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/242513877' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:31.919 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:01:31] "GET /metrics HTTP/1.1" 200 36509 "" "Prometheus/2.43.0" 2024-04-05T03:01:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:31 smithi150 ceph-mon[107701]: pgmap v193: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 53 KiB/s wr, 606 op/s; 8899/351864 objects degraded (2.529%); 2.5 MiB/s, 40 objects/s recovering 2024-04-05T03:01:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:31 smithi150 ceph-mon[103445]: pgmap v193: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 53 KiB/s wr, 606 op/s; 8899/351864 objects degraded (2.529%); 2.5 MiB/s, 40 objects/s recovering 2024-04-05T03:01:32.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:31 smithi196 ceph-mon[91352]: pgmap v193: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 53 KiB/s wr, 606 op/s; 8899/351864 objects degraded (2.529%); 2.5 MiB/s, 40 objects/s recovering 2024-04-05T03:01:33.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8899/351864 objects degraded (2.529%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:33.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:32 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1797127531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8899/351864 objects degraded (2.529%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:33.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:32 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1797127531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:33.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8899/351864 objects degraded (2.529%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:33.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:32 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1797127531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:34.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:33 smithi150 ceph-mon[103445]: pgmap v194: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 53 KiB/s wr, 926 op/s; 8669/340341 objects degraded (2.547%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:01:34.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:01:33.990+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:34.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:33 smithi150 ceph-mon[107701]: pgmap v194: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 53 KiB/s wr, 926 op/s; 8669/340341 objects degraded (2.547%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:01:34.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:33 smithi196 ceph-mon[91352]: pgmap v194: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 85 B/s rd, 53 KiB/s wr, 926 op/s; 8669/340341 objects degraded (2.547%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:01:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:35.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:35.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2575017294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:35.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:34 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2575017294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:35.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:35.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:34 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:35.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:34 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:35.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:34 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2575017294' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:36.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:35 smithi150 ceph-mon[103445]: pgmap v195: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8669/340341 objects degraded (2.547%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:01:36.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:35 smithi150 ceph-mon[107701]: pgmap v195: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8669/340341 objects degraded (2.547%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:01:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:35 smithi196 ceph-mon[91352]: pgmap v195: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 8669/340341 objects degraded (2.547%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:01:38.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:37 smithi150 ceph-mon[103445]: pgmap v196: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 8669/336126 objects degraded (2.579%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:01:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8669/336126 objects degraded (2.579%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:38.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:37 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1900398952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:37 smithi150 ceph-mon[107701]: pgmap v196: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 8669/336126 objects degraded (2.579%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:01:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8669/336126 objects degraded (2.579%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:38.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:37 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1900398952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:38.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:37 smithi196 ceph-mon[91352]: pgmap v196: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 8669/336126 objects degraded (2.579%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:01:38.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8669/336126 objects degraded (2.579%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:38.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:37 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1900398952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:40.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:39 smithi150 ceph-mon[107701]: pgmap v197: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 8435/328920 objects degraded (2.564%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:01:40.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:39 smithi150 ceph-mon[103445]: pgmap v197: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 8435/328920 objects degraded (2.564%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:01:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:39 smithi196 ceph-mon[91352]: pgmap v197: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 8435/328920 objects degraded (2.564%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:01:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2144884382' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:40.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2144884382' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:40.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:41.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2144884382' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:41.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:41.976 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:01:41] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2024-04-05T03:01:42.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:41 smithi150 ceph-mon[103445]: pgmap v198: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 8435/328920 objects degraded (2.564%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:01:42.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:41 smithi150 ceph-mon[107701]: pgmap v198: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 8435/328920 objects degraded (2.564%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:01:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:41 smithi196 ceph-mon[91352]: pgmap v198: 89 pgs: 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 8435/328920 objects degraded (2.564%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:01:43.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:42 smithi196 ceph-mon[91352]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:42 smithi196 ceph-mon[91352]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:42 smithi196 ceph-mon[91352]: osdmap e97: 8 total, 8 up, 8 in 2024-04-05T03:01:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:42 smithi196 ceph-mon[91352]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:42 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8435/328920 objects degraded (2.564%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:42 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1268874740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:43.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[103445]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[103445]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[103445]: osdmap e97: 8 total, 8 up, 8 in 2024-04-05T03:01:43.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[103445]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8435/328920 objects degraded (2.564%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:43.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1268874740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:43.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[107701]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[107701]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[107701]: osdmap e97: 8 total, 8 up, 8 in 2024-04-05T03:01:43.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[107701]: failed to encode map e97 with expected crc 2024-04-05T03:01:43.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8435/328920 objects degraded (2.564%), 8 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:43.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:42 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1268874740' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:43 smithi196 ceph-mon[91352]: pgmap v200: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8206/318483 objects degraded (2.577%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:01:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:43 smithi196 ceph-mon[91352]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:43 smithi196 ceph-mon[91352]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:43 smithi196 ceph-mon[91352]: osdmap e98: 8 total, 8 up, 8 in 2024-04-05T03:01:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:43 smithi196 ceph-mon[91352]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[107701]: pgmap v200: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8206/318483 objects degraded (2.577%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[107701]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[107701]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[107701]: osdmap e98: 8 total, 8 up, 8 in 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[107701]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[103445]: pgmap v200: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 8206/318483 objects degraded (2.577%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[103445]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[103445]: failed to encode map e98 with expected crc 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[103445]: osdmap e98: 8 total, 8 up, 8 in 2024-04-05T03:01:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:43 smithi150 ceph-mon[103445]: failed to encode map e98 with expected crc 2024-04-05T03:01:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:45 smithi196 ceph-mon[91352]: pgmap v202: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8206/318483 objects degraded (2.577%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:01:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:45 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/207427729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:46 smithi150 ceph-mon[107701]: pgmap v202: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8206/318483 objects degraded (2.577%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:01:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:46 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/207427729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:46.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:46 smithi150 ceph-mon[103445]: pgmap v202: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 8206/318483 objects degraded (2.577%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:01:46.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:46 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/207427729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:47.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8206/318483 objects degraded (2.577%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8206/318483 objects degraded (2.577%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:47.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8206/318483 objects degraded (2.577%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:48.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:48 smithi196 ceph-mon[91352]: pgmap v203: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 8206/315129 objects degraded (2.604%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:01:48.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:48 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1883880709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:48 smithi150 ceph-mon[107701]: pgmap v203: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 8206/315129 objects degraded (2.604%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:01:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:48 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1883880709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:48 smithi150 ceph-mon[103445]: pgmap v203: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 8206/315129 objects degraded (2.604%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:01:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:48 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1883880709' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:49.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:49 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:49.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:49 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:49.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:01:49.007+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:49.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:49 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:50.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:50 smithi196 ceph-mon[91352]: pgmap v204: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 7833/309210 objects degraded (2.533%); 4.7 MiB/s, 75 objects/s recovering 2024-04-05T03:01:50.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:50 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:50.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:50 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:50 smithi150 ceph-mon[107701]: pgmap v204: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 7833/309210 objects degraded (2.533%); 4.7 MiB/s, 75 objects/s recovering 2024-04-05T03:01:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:50 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:50 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:50 smithi150 ceph-mon[103445]: pgmap v204: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 7833/309210 objects degraded (2.533%); 4.7 MiB/s, 75 objects/s recovering 2024-04-05T03:01:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:50 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:01:50.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:50 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:01:51.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:51 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2356185949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:51.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:51 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2356185949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:51 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2356185949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:01:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:01:51] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2024-04-05T03:01:52.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:52 smithi196 ceph-mon[91352]: pgmap v205: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 7833/309210 objects degraded (2.533%); 4.1 MiB/s, 66 objects/s recovering 2024-04-05T03:01:52.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7833/309210 objects degraded (2.533%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:52.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:52 smithi150 ceph-mon[107701]: pgmap v205: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 7833/309210 objects degraded (2.533%); 4.1 MiB/s, 66 objects/s recovering 2024-04-05T03:01:52.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7833/309210 objects degraded (2.533%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:52 smithi150 ceph-mon[103445]: pgmap v205: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.3 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 518 op/s; 7833/309210 objects degraded (2.533%); 4.1 MiB/s, 66 objects/s recovering 2024-04-05T03:01:52.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7833/309210 objects degraded (2.533%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:53.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:53 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2198043215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:53.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:53 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2198043215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:53.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:53 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2198043215' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:54.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:54 smithi196 ceph-mon[91352]: pgmap v206: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 7606/298155 objects degraded (2.551%); 3.7 MiB/s, 60 objects/s recovering 2024-04-05T03:01:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:54 smithi150 ceph-mon[107701]: pgmap v206: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 7606/298155 objects degraded (2.551%); 3.7 MiB/s, 60 objects/s recovering 2024-04-05T03:01:54.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:54 smithi150 ceph-mon[103445]: pgmap v206: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 7606/298155 objects degraded (2.551%); 3.7 MiB/s, 60 objects/s recovering 2024-04-05T03:01:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:56 smithi196 ceph-mon[91352]: pgmap v207: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 7606/298155 objects degraded (2.551%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:01:56.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:56 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4139327377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:56.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:56 smithi150 ceph-mon[107701]: pgmap v207: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 7606/298155 objects degraded (2.551%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:01:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:56 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4139327377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:56.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:56 smithi150 ceph-mon[103445]: pgmap v207: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 573 op/s; 7606/298155 objects degraded (2.551%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:01:56.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:56 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4139327377' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:56.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:01:57.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7606/298155 objects degraded (2.551%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:57.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7606/298155 objects degraded (2.551%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:57.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7606/298155 objects degraded (2.551%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:01:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:58 smithi196 ceph-mon[91352]: pgmap v208: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 7606/294009 objects degraded (2.587%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:01:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:01:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3623958536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:58 smithi150 ceph-mon[107701]: pgmap v208: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 7606/294009 objects degraded (2.587%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:01:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:01:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3623958536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:01:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:58 smithi150 ceph-mon[103445]: pgmap v208: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 7606/294009 objects degraded (2.587%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:01:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:01:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3623958536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:00 smithi196 ceph-mon[91352]: pgmap v209: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 7380/287070 objects degraded (2.571%); 4.3 MiB/s, 68 objects/s recovering 2024-04-05T03:02:00.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:00 smithi150 ceph-mon[107701]: pgmap v209: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 7380/287070 objects degraded (2.571%); 4.3 MiB/s, 68 objects/s recovering 2024-04-05T03:02:00.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:00 smithi150 ceph-mon[103445]: pgmap v209: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 7380/287070 objects degraded (2.571%); 4.3 MiB/s, 68 objects/s recovering 2024-04-05T03:02:01.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:01 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/539489167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:01.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:01 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/539489167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:01.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:01 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/539489167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:02:01] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2024-04-05T03:02:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:02 smithi196 ceph-mon[91352]: pgmap v210: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 7380/287070 objects degraded (2.571%); 2.4 MiB/s, 37 objects/s recovering 2024-04-05T03:02:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7380/287070 objects degraded (2.571%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:02 smithi150 ceph-mon[103445]: pgmap v210: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 7380/287070 objects degraded (2.571%); 2.4 MiB/s, 37 objects/s recovering 2024-04-05T03:02:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7380/287070 objects degraded (2.571%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:02.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:02 smithi150 ceph-mon[107701]: pgmap v210: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 7380/287070 objects degraded (2.571%); 2.4 MiB/s, 37 objects/s recovering 2024-04-05T03:02:02.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7380/287070 objects degraded (2.571%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:04.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: pgmap v211: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 7121/275982 objects degraded (2.580%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:02:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: osdmap e99: 8 total, 8 up, 8 in 2024-04-05T03:02:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1348896624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:04.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:04.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: pgmap v211: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 7121/275982 objects degraded (2.580%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:02:04.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: osdmap e99: 8 total, 8 up, 8 in 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1348896624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:02:04.020+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: pgmap v211: 89 pgs: 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 922 op/s; 7121/275982 objects degraded (2.580%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: osdmap e99: 8 total, 8 up, 8 in 2024-04-05T03:02:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: failed to encode map e99 with expected crc 2024-04-05T03:02:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1348896624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:04.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:05.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:05.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:02:05.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: osdmap e100: 8 total, 8 up, 8 in 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[107701]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: osdmap e100: 8 total, 8 up, 8 in 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:05 smithi150 ceph-mon[103445]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:02:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: osdmap e100: 8 total, 8 up, 8 in 2024-04-05T03:02:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: failed to encode map e100 with expected crc 2024-04-05T03:02:05.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:05 smithi196 ceph-mon[91352]: failed to encode map e100 with expected crc 2024-04-05T03:02:06.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:06 smithi150 ceph-mon[103445]: pgmap v214: 89 pgs: 1 peering, 81 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 7113/275856 objects degraded (2.579%); 3.9 MiB/s, 61 objects/s recovering 2024-04-05T03:02:06.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:06 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3894152013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:06.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:06 smithi150 ceph-mon[107701]: pgmap v214: 89 pgs: 1 peering, 81 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 7113/275856 objects degraded (2.579%); 3.9 MiB/s, 61 objects/s recovering 2024-04-05T03:02:06.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:06 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3894152013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:06 smithi196 ceph-mon[91352]: pgmap v214: 89 pgs: 1 peering, 81 active+clean, 1 active+recovery_wait, 6 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 7113/275856 objects degraded (2.579%); 3.9 MiB/s, 61 objects/s recovering 2024-04-05T03:02:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:06 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3894152013' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:07.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7113/275856 objects degraded (2.579%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:07.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7113/275856 objects degraded (2.579%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7113/275856 objects degraded (2.579%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:08.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:08 smithi150 ceph-mon[103445]: pgmap v215: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6752/272256 objects degraded (2.480%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:02:08.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:08 smithi150 ceph-mon[107701]: pgmap v215: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6752/272256 objects degraded (2.480%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:02:08.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:08 smithi196 ceph-mon[91352]: pgmap v215: 89 pgs: 1 peering, 81 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 6752/272256 objects degraded (2.480%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:02:09.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:09 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1089542282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:09.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:09 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1089542282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:09.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:09 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1089542282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:10.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:10 smithi150 ceph-mon[107701]: pgmap v216: 89 pgs: 1 peering, 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 6752/267048 objects degraded (2.528%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:02:10.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:10 smithi150 ceph-mon[103445]: pgmap v216: 89 pgs: 1 peering, 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 6752/267048 objects degraded (2.528%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:02:10.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:10 smithi196 ceph-mon[91352]: pgmap v216: 89 pgs: 1 peering, 81 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 834 op/s; 6752/267048 objects degraded (2.528%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:02:11.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:11.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:11 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/571322422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:11.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:11 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/571322422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:11 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/571322422' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:11.809 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:02:11] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2024-04-05T03:02:12.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:12 smithi150 ceph-mon[107701]: pgmap v217: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 6752/266289 objects degraded (2.536%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:02:12.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 6752/266289 objects degraded (2.536%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:12.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:12 smithi150 ceph-mon[103445]: pgmap v217: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 6752/266289 objects degraded (2.536%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:02:12.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 6752/266289 objects degraded (2.536%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:12.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:12 smithi196 ceph-mon[91352]: pgmap v217: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 403 op/s; 6752/266289 objects degraded (2.536%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:02:12.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 6752/266289 objects degraded (2.536%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:14 smithi150 ceph-mon[103445]: pgmap v218: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6501/256092 objects degraded (2.539%); 4.0 MiB/s, 63 objects/s recovering 2024-04-05T03:02:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:14 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2615820314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:14 smithi150 ceph-mon[107701]: pgmap v218: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6501/256092 objects degraded (2.539%); 4.0 MiB/s, 63 objects/s recovering 2024-04-05T03:02:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:14 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2615820314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:14 smithi196 ceph-mon[91352]: pgmap v218: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 6501/256092 objects degraded (2.539%); 4.0 MiB/s, 63 objects/s recovering 2024-04-05T03:02:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:14 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2615820314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:16.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:16 smithi150 ceph-mon[103445]: pgmap v219: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6501/255456 objects degraded (2.545%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:02:16.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:16 smithi150 ceph-mon[107701]: pgmap v219: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6501/255456 objects degraded (2.545%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:02:16.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:16 smithi196 ceph-mon[91352]: pgmap v219: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 635 op/s; 6501/255456 objects degraded (2.545%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:02:17.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2244336354' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:17.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 6501/255456 objects degraded (2.545%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:17.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2244336354' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:17.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 6501/255456 objects degraded (2.545%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2244336354' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 6501/255456 objects degraded (2.545%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:02:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:18 smithi150 ceph-mon[103445]: pgmap v220: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 6257/251526 objects degraded (2.488%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:02:18.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:18 smithi150 ceph-mon[107701]: pgmap v220: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 6257/251526 objects degraded (2.488%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:02:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:18 smithi196 ceph-mon[91352]: pgmap v220: 89 pgs: 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 6257/251526 objects degraded (2.488%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:02:19.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:19 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4283331497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:19.641 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:19 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4283331497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:19 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4283331497' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:20.561 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:20 smithi150 ceph-mon[103445]: pgmap v221: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6257/245358 objects degraded (2.550%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:02:20.561 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:20 smithi150 ceph-mon[107701]: pgmap v221: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6257/245358 objects degraded (2.550%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:02:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:20 smithi196 ceph-mon[91352]: pgmap v221: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 6257/245358 objects degraded (2.550%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:02:21.812 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:02:21] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2024-04-05T03:02:22.505 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:22 smithi150 ceph-mon[103445]: pgmap v222: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 6257/244536 objects degraded (2.559%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:02:22.506 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1284284480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:22.506 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 6257/244536 objects degraded (2.559%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:22.506 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:22 smithi150 ceph-mon[107701]: pgmap v222: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 6257/244536 objects degraded (2.559%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:02:22.506 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1284284480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:22.506 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 6257/244536 objects degraded (2.559%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:22.590 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:22 smithi196 ceph-mon[91352]: pgmap v222: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 6257/244536 objects degraded (2.559%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:02:22.591 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1284284480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:22.591 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 6257/244536 objects degraded (2.559%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:24.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: pgmap v223: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 5975/234972 objects degraded (2.543%); 4.0 MiB/s, 64 objects/s recovering 2024-04-05T03:02:24.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: osdmap e101: 8 total, 8 up, 8 in 2024-04-05T03:02:24.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:24 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/75825190' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:24.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: pgmap v223: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 5975/234972 objects degraded (2.543%); 4.0 MiB/s, 64 objects/s recovering 2024-04-05T03:02:24.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: osdmap e101: 8 total, 8 up, 8 in 2024-04-05T03:02:24.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/75825190' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: pgmap v223: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 5975/234972 objects degraded (2.543%); 4.0 MiB/s, 64 objects/s recovering 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: osdmap e101: 8 total, 8 up, 8 in 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: failed to encode map e101 with expected crc 2024-04-05T03:02:24.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:24 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/75825190' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:25.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: osdmap e102: 8 total, 8 up, 8 in 2024-04-05T03:02:25.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: osdmap e102: 8 total, 8 up, 8 in 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.539 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: osdmap e102: 8 total, 8 up, 8 in 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: failed to encode map e102 with expected crc 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:25 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:25.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:25 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:02:25.537+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: pgmap v226: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 5975/234381 objects degraded (2.549%); 2.2 MiB/s, 35 objects/s recovering 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: pgmap v227: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5975/234381 objects degraded (2.549%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: pgmap v228: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5975/234381 objects degraded (2.549%); 3.9 MiB/s, 62 objects/s recovering 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:26.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:26.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:26.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:26.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: pgmap v226: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 5975/234381 objects degraded (2.549%); 2.2 MiB/s, 35 objects/s recovering 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: pgmap v227: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5975/234381 objects degraded (2.549%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: pgmap v228: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5975/234381 objects degraded (2.549%); 3.9 MiB/s, 62 objects/s recovering 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:26.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: pgmap v226: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 5975/234381 objects degraded (2.549%); 2.2 MiB/s, 35 objects/s recovering 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: pgmap v227: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 5975/234381 objects degraded (2.549%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: pgmap v228: 89 pgs: 82 active+clean, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5975/234381 objects degraded (2.549%); 3.9 MiB/s, 62 objects/s recovering 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:02:26.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:26.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:26.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:26.730 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:27 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1444285650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 5975/234381 objects degraded (2.549%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:27 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1444285650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 5975/234381 objects degraded (2.549%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:27.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:27 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1444285650' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:27.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 5975/234381 objects degraded (2.549%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:28.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:28 smithi196 ceph-mon[91352]: pgmap v229: 89 pgs: 83 active+clean, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 5928/228753 objects degraded (2.591%); 665 KiB/s, 10 objects/s recovering 2024-04-05T03:02:28.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:28 smithi150 ceph-mon[107701]: pgmap v229: 89 pgs: 83 active+clean, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 5928/228753 objects degraded (2.591%); 665 KiB/s, 10 objects/s recovering 2024-04-05T03:02:28.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:28 smithi150 ceph-mon[103445]: pgmap v229: 89 pgs: 83 active+clean, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 5928/228753 objects degraded (2.591%); 665 KiB/s, 10 objects/s recovering 2024-04-05T03:02:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1382819646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1382819646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:29.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1382819646' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:30.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:30 smithi196 ceph-mon[91352]: pgmap v230: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 5650/225294 objects degraded (2.508%); 3.3 MiB/s, 53 objects/s recovering 2024-04-05T03:02:30.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:30 smithi150 ceph-mon[107701]: pgmap v230: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 5650/225294 objects degraded (2.508%); 3.3 MiB/s, 53 objects/s recovering 2024-04-05T03:02:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:30 smithi150 ceph-mon[103445]: pgmap v230: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 5650/225294 objects degraded (2.508%); 3.3 MiB/s, 53 objects/s recovering 2024-04-05T03:02:31.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:02:31] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2024-04-05T03:02:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:32 smithi196 ceph-mon[91352]: pgmap v231: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 5650/225294 objects degraded (2.508%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:02:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:32 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1739606602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:32.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 5650/225294 objects degraded (2.508%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:32 smithi150 ceph-mon[103445]: pgmap v231: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 5650/225294 objects degraded (2.508%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:02:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:32 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1739606602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 5650/225294 objects degraded (2.508%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:32.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:32 smithi150 ceph-mon[107701]: pgmap v231: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 427 op/s; 5650/225294 objects degraded (2.508%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:02:32.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:32 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1739606602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:32.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 5650/225294 objects degraded (2.508%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:34 smithi196 ceph-mon[91352]: pgmap v232: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5487/214905 objects degraded (2.553%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:02:34.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:34 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4098350015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:34.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:34 smithi150 ceph-mon[103445]: pgmap v232: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5487/214905 objects degraded (2.553%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:02:34.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:34 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4098350015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:34.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:34 smithi150 ceph-mon[107701]: pgmap v232: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 761 op/s; 5487/214905 objects degraded (2.553%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:02:34.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:34 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4098350015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:36 smithi196 ceph-mon[91352]: pgmap v233: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5487/214203 objects degraded (2.562%); 3.0 MiB/s, 48 objects/s recovering 2024-04-05T03:02:36.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:36 smithi150 ceph-mon[107701]: pgmap v233: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5487/214203 objects degraded (2.562%); 3.0 MiB/s, 48 objects/s recovering 2024-04-05T03:02:36.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:36 smithi150 ceph-mon[103445]: pgmap v233: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 5487/214203 objects degraded (2.562%); 3.0 MiB/s, 48 objects/s recovering 2024-04-05T03:02:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:37 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/91617510' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 5487/214203 objects degraded (2.562%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:37 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/91617510' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 5487/214203 objects degraded (2.562%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:37.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:37 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/91617510' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 5487/214203 objects degraded (2.562%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:38 smithi196 ceph-mon[91352]: pgmap v234: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5487/208626 objects degraded (2.630%); 2.5 MiB/s, 40 objects/s recovering 2024-04-05T03:02:38.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:38 smithi150 ceph-mon[103445]: pgmap v234: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5487/208626 objects degraded (2.630%); 2.5 MiB/s, 40 objects/s recovering 2024-04-05T03:02:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:38 smithi150 ceph-mon[107701]: pgmap v234: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 5487/208626 objects degraded (2.630%); 2.5 MiB/s, 40 objects/s recovering 2024-04-05T03:02:40.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:40 smithi150 ceph-mon[107701]: pgmap v235: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5322/203067 objects degraded (2.621%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:02:40.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3305310108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:40.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:02:40.557+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:40.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:40 smithi150 ceph-mon[103445]: pgmap v235: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5322/203067 objects degraded (2.621%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:02:40.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3305310108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:40.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:40 smithi196 ceph-mon[91352]: pgmap v235: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 5322/203067 objects degraded (2.621%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:02:40.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3305310108' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:41.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:41 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:41 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:41.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:02:41] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2024-04-05T03:02:42.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:42 smithi150 ceph-mon[107701]: pgmap v236: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5322/203067 objects degraded (2.621%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:02:42.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:42 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 5322/203067 objects degraded (2.621%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:42.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:42 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1337161713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:42.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:42 smithi150 ceph-mon[103445]: pgmap v236: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5322/203067 objects degraded (2.621%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:02:42.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:42 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 5322/203067 objects degraded (2.621%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:42.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:42 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1337161713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:42.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:42 smithi196 ceph-mon[91352]: pgmap v236: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 5322/203067 objects degraded (2.621%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:02:42.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:42 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 5322/203067 objects degraded (2.621%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:42.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:42 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1337161713' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:44.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:44 smithi150 ceph-mon[107701]: pgmap v237: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 5143/193176 objects degraded (2.662%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:02:44.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:44 smithi150 ceph-mon[103445]: pgmap v237: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 5143/193176 objects degraded (2.662%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:02:44.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:44 smithi196 ceph-mon[91352]: pgmap v237: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 5143/193176 objects degraded (2.662%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:02:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:45 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/388812698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:45.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:45 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/388812698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:45 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/388812698' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:46.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:46 smithi150 ceph-mon[107701]: pgmap v238: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5143/192465 objects degraded (2.672%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:02:46.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:46 smithi150 ceph-mon[103445]: pgmap v238: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5143/192465 objects degraded (2.672%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:02:46.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:46 smithi196 ceph-mon[91352]: pgmap v238: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 5143/192465 objects degraded (2.672%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:02:47.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 5143/192465 objects degraded (2.672%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:47.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:47 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/697759006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:47.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 5143/192465 objects degraded (2.672%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:47.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:47 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/697759006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:47.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 5143/192465 objects degraded (2.672%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:47.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:47 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/697759006' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:48.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:48 smithi150 ceph-mon[103445]: pgmap v239: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5143/186909 objects degraded (2.752%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:02:48.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:48 smithi150 ceph-mon[107701]: pgmap v239: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5143/186909 objects degraded (2.752%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:02:48.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:48 smithi196 ceph-mon[91352]: pgmap v239: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 757 op/s; 5143/186909 objects degraded (2.752%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:02:50.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:50 smithi150 ceph-mon[107701]: pgmap v240: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 4959/181524 objects degraded (2.732%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:02:50.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:50 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3186937902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:50.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:50 smithi150 ceph-mon[103445]: pgmap v240: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 4959/181524 objects degraded (2.732%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:02:50.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:50 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3186937902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:50.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:50 smithi196 ceph-mon[91352]: pgmap v240: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 752 op/s; 4959/181524 objects degraded (2.732%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:02:50.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:50 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3186937902' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:02:51] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2024-04-05T03:02:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:52 smithi150 ceph-mon[103445]: pgmap v241: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 4959/181524 objects degraded (2.732%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:02:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4959/181524 objects degraded (2.732%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1968234568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:52 smithi150 ceph-mon[107701]: pgmap v241: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 4959/181524 objects degraded (2.732%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:02:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4959/181524 objects degraded (2.732%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1968234568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:52 smithi196 ceph-mon[91352]: pgmap v241: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 4959/181524 objects degraded (2.732%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:02:52.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4959/181524 objects degraded (2.732%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:52.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1968234568' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[107701]: pgmap v242: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4770/171207 objects degraded (2.786%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:02:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[107701]: osdmap e103: 8 total, 8 up, 8 in 2024-04-05T03:02:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[107701]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[107701]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[107701]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[107701]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[103445]: pgmap v242: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4770/171207 objects degraded (2.786%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[103445]: osdmap e103: 8 total, 8 up, 8 in 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[103445]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[103445]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[103445]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:54 smithi150 ceph-mon[103445]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:54 smithi196 ceph-mon[91352]: pgmap v242: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4770/171207 objects degraded (2.786%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:02:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:54 smithi196 ceph-mon[91352]: osdmap e103: 8 total, 8 up, 8 in 2024-04-05T03:02:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:54 smithi196 ceph-mon[91352]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:54 smithi196 ceph-mon[91352]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:54 smithi196 ceph-mon[91352]: failed to encode map e103 with expected crc 2024-04-05T03:02:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:54 smithi196 ceph-mon[91352]: failed to encode map e103 with expected crc 2024-04-05T03:02:55.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: osdmap e104: 8 total, 8 up, 8 in 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1982103426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:02:55.573+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: osdmap e104: 8 total, 8 up, 8 in 2024-04-05T03:02:55.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:55 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1982103426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:55.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: osdmap e104: 8 total, 8 up, 8 in 2024-04-05T03:02:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: failed to encode map e104 with expected crc 2024-04-05T03:02:55.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:55 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1982103426' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:56 smithi196 ceph-mon[91352]: pgmap v245: 89 pgs: 1 peering, 83 active+clean, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4748/170487 objects degraded (2.785%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:02:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:56 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:56 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[107701]: pgmap v245: 89 pgs: 1 peering, 83 active+clean, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4748/170487 objects degraded (2.785%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:02:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:56.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:56.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[103445]: pgmap v245: 89 pgs: 1 peering, 83 active+clean, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 4748/170487 objects degraded (2.785%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:02:56.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:56.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:02:56.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:02:56.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:02:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4748/170487 objects degraded (2.785%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:57.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4748/170487 objects degraded (2.785%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4748/170487 objects degraded (2.785%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:02:58.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:58 smithi196 ceph-mon[91352]: pgmap v246: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 4481/165711 objects degraded (2.704%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:02:58.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:02:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1185774679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:58.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:58 smithi150 ceph-mon[107701]: pgmap v246: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 4481/165711 objects degraded (2.704%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:02:58.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:02:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1185774679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:02:58.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:58 smithi150 ceph-mon[103445]: pgmap v246: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 4481/165711 objects degraded (2.704%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:02:58.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:02:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1185774679' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:00.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:00 smithi196 ceph-mon[91352]: pgmap v247: 89 pgs: 83 active+clean, 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4481/160926 objects degraded (2.785%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:03:00.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:00 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1415499489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:00 smithi150 ceph-mon[107701]: pgmap v247: 89 pgs: 83 active+clean, 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4481/160926 objects degraded (2.785%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:03:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:00 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1415499489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:00.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:00 smithi150 ceph-mon[103445]: pgmap v247: 89 pgs: 83 active+clean, 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4481/160926 objects degraded (2.785%); 3.7 MiB/s, 59 objects/s recovering 2024-04-05T03:03:00.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:00 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1415499489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:03:01] "GET /metrics HTTP/1.1" 200 36503 "" "Prometheus/2.43.0" 2024-04-05T03:03:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:02 smithi196 ceph-mon[91352]: pgmap v248: 89 pgs: 83 active+clean, 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 4481/160926 objects degraded (2.785%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:03:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4481/160926 objects degraded (2.785%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:02.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:02 smithi150 ceph-mon[103445]: pgmap v248: 89 pgs: 83 active+clean, 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 4481/160926 objects degraded (2.785%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:03:02.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4481/160926 objects degraded (2.785%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:02 smithi150 ceph-mon[107701]: pgmap v248: 89 pgs: 83 active+clean, 2 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 4481/160926 objects degraded (2.785%); 2.3 MiB/s, 36 objects/s recovering 2024-04-05T03:03:02.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4481/160926 objects degraded (2.785%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:03 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2945842419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:03.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:03 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2945842419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:03.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:03 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2945842419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:04 smithi196 ceph-mon[91352]: pgmap v249: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 4252/151215 objects degraded (2.812%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:03:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:04 smithi150 ceph-mon[103445]: pgmap v249: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 4252/151215 objects degraded (2.812%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:03:04.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:04 smithi150 ceph-mon[107701]: pgmap v249: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 674 op/s; 4252/151215 objects degraded (2.812%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:03:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2280551077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2280551077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2280551077' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:06.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:06 smithi196 ceph-mon[91352]: pgmap v250: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4252/149814 objects degraded (2.838%); 2.8 MiB/s, 45 objects/s recovering 2024-04-05T03:03:06.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:06 smithi150 ceph-mon[103445]: pgmap v250: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4252/149814 objects degraded (2.838%); 2.8 MiB/s, 45 objects/s recovering 2024-04-05T03:03:06.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:06 smithi150 ceph-mon[107701]: pgmap v250: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 4252/149814 objects degraded (2.838%); 2.8 MiB/s, 45 objects/s recovering 2024-04-05T03:03:07.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4252/149814 objects degraded (2.838%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4252/149814 objects degraded (2.838%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:07.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4252/149814 objects degraded (2.838%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:08.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:08 smithi196 ceph-mon[91352]: pgmap v251: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 4011/144330 objects degraded (2.779%); 3.8 MiB/s, 61 objects/s recovering 2024-04-05T03:03:08.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:08 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/333457723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:08.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:08 smithi150 ceph-mon[103445]: pgmap v251: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 4011/144330 objects degraded (2.779%); 3.8 MiB/s, 61 objects/s recovering 2024-04-05T03:03:08.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:08 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/333457723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:08.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:08 smithi150 ceph-mon[107701]: pgmap v251: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 4011/144330 objects degraded (2.779%); 3.8 MiB/s, 61 objects/s recovering 2024-04-05T03:03:08.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:08 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/333457723' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:10.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:10 smithi196 ceph-mon[91352]: pgmap v252: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 4011/138729 objects degraded (2.891%); 2.4 MiB/s, 39 objects/s recovering 2024-04-05T03:03:10.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1499835913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:10 smithi150 ceph-mon[103445]: pgmap v252: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 4011/138729 objects degraded (2.891%); 2.4 MiB/s, 39 objects/s recovering 2024-04-05T03:03:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1499835913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:10.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:10 smithi150 ceph-mon[107701]: pgmap v252: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 4011/138729 objects degraded (2.891%); 2.4 MiB/s, 39 objects/s recovering 2024-04-05T03:03:10.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1499835913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:10.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:03:10.590+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:11.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:11.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:11.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:11.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:11.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:03:11] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2024-04-05T03:03:12.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:12 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:12.889 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:12 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:12.890 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:12 smithi196 ceph-mon[91352]: pgmap v253: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 4011/138729 objects degraded (2.891%); 2.4 MiB/s, 39 objects/s recovering 2024-04-05T03:03:12.890 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4011/138729 objects degraded (2.891%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[103445]: pgmap v253: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 4011/138729 objects degraded (2.891%); 2.4 MiB/s, 39 objects/s recovering 2024-04-05T03:03:12.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4011/138729 objects degraded (2.891%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[107701]: pgmap v253: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 4011/138729 objects degraded (2.891%); 2.4 MiB/s, 39 objects/s recovering 2024-04-05T03:03:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4011/138729 objects degraded (2.891%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:13.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:13 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/379884782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:13.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:13 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/379884782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:13.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:13 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/379884782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:14 smithi196 ceph-mon[91352]: pgmap v254: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3760/129018 objects degraded (2.914%); 3.8 MiB/s, 60 objects/s recovering 2024-04-05T03:03:14.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:14 smithi150 ceph-mon[103445]: pgmap v254: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3760/129018 objects degraded (2.914%); 3.8 MiB/s, 60 objects/s recovering 2024-04-05T03:03:14.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:14 smithi150 ceph-mon[107701]: pgmap v254: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 3760/129018 objects degraded (2.914%); 3.8 MiB/s, 60 objects/s recovering 2024-04-05T03:03:16.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:16 smithi196 ceph-mon[91352]: pgmap v255: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 3760/127581 objects degraded (2.947%); 2.6 MiB/s, 40 objects/s recovering 2024-04-05T03:03:16.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:16 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1909028429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:16 smithi150 ceph-mon[103445]: pgmap v255: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 3760/127581 objects degraded (2.947%); 2.6 MiB/s, 40 objects/s recovering 2024-04-05T03:03:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:16 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1909028429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:16.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:16 smithi150 ceph-mon[107701]: pgmap v255: 89 pgs: 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 656 op/s; 3760/127581 objects degraded (2.947%); 2.6 MiB/s, 40 objects/s recovering 2024-04-05T03:03:16.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:16 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1909028429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:17.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: osdmap e105: 8 total, 8 up, 8 in 2024-04-05T03:03:17.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 3760/127581 objects degraded (2.947%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: osdmap e105: 8 total, 8 up, 8 in 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 3760/127581 objects degraded (2.947%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: osdmap e105: 8 total, 8 up, 8 in 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: failed to encode map e105 with expected crc 2024-04-05T03:03:17.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 3760/127581 objects degraded (2.947%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:18.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: pgmap v257: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 3483/122151 objects degraded (2.851%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:03:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: osdmap e106: 8 total, 8 up, 8 in 2024-04-05T03:03:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:18 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3181739879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:18.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: pgmap v257: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 3483/122151 objects degraded (2.851%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: osdmap e106: 8 total, 8 up, 8 in 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3181739879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: pgmap v257: 89 pgs: 1 peering, 83 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 739 op/s; 3483/122151 objects degraded (2.851%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: osdmap e106: 8 total, 8 up, 8 in 2024-04-05T03:03:18.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: failed to encode map e106 with expected crc 2024-04-05T03:03:18.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:18 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3181739879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:19.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:19 smithi196 ceph-mon[91352]: pgmap v259: 89 pgs: 1 peering, 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 3483/117336 objects degraded (2.968%); 4.1 MiB/s, 65 objects/s recovering 2024-04-05T03:03:19.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:19 smithi150 ceph-mon[103445]: pgmap v259: 89 pgs: 1 peering, 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 3483/117336 objects degraded (2.968%); 4.1 MiB/s, 65 objects/s recovering 2024-04-05T03:03:19.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:19 smithi150 ceph-mon[107701]: pgmap v259: 89 pgs: 1 peering, 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 891 op/s; 3483/117336 objects degraded (2.968%); 4.1 MiB/s, 65 objects/s recovering 2024-04-05T03:03:21.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:20 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1436264615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:21.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:20 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1436264615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:21.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:20 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1436264615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:21.948 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:03:21] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2024-04-05T03:03:22.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:21 smithi150 ceph-mon[107701]: pgmap v260: 89 pgs: 1 peering, 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 3483/117336 objects degraded (2.968%); 2.2 MiB/s, 34 objects/s recovering 2024-04-05T03:03:22.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:21 smithi150 ceph-mon[103445]: pgmap v260: 89 pgs: 1 peering, 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 3483/117336 objects degraded (2.968%); 2.2 MiB/s, 34 objects/s recovering 2024-04-05T03:03:22.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:21 smithi196 ceph-mon[91352]: pgmap v260: 89 pgs: 1 peering, 85 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 486 op/s; 3483/117336 objects degraded (2.968%); 2.2 MiB/s, 34 objects/s recovering 2024-04-05T03:03:23.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 3483/117336 objects degraded (2.968%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 3483/117336 objects degraded (2.968%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:23.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 3483/117336 objects degraded (2.968%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:23 smithi196 ceph-mon[91352]: pgmap v261: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 3184/108612 objects degraded (2.932%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:03:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:23 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3208502145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:24.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:23 smithi150 ceph-mon[107701]: pgmap v261: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 3184/108612 objects degraded (2.932%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:03:24.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:23 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3208502145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:24.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:23 smithi150 ceph-mon[103445]: pgmap v261: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 3184/108612 objects degraded (2.932%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:03:24.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:23 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3208502145' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[107701]: pgmap v262: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3184/107334 objects degraded (2.966%); 4.1 MiB/s, 65 objects/s recovering 2024-04-05T03:03:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:26.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:26.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2105506186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:26.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[103445]: pgmap v262: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3184/107334 objects degraded (2.966%); 4.1 MiB/s, 65 objects/s recovering 2024-04-05T03:03:26.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:26.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:26.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:26 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2105506186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:26.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:26 smithi196 ceph-mon[91352]: pgmap v262: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 3184/107334 objects degraded (2.966%); 4.1 MiB/s, 65 objects/s recovering 2024-04-05T03:03:26.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:26.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:26.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:26 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2105506186' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:27.718 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 3184/107334 objects degraded (2.966%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:27.719 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 3184/107334 objects degraded (2.966%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:27.789 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 3184/107334 objects degraded (2.966%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:28.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:28 smithi150 ceph-mon[107701]: pgmap v263: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2982/101952 objects degraded (2.925%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:03:28.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:28 smithi150 ceph-mon[103445]: pgmap v263: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2982/101952 objects degraded (2.925%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:03:28.800 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:28 smithi196 ceph-mon[91352]: pgmap v263: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2982/101952 objects degraded (2.925%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:03:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4126703792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:29.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4126703792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:29.795 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4126703792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:30 smithi150 ceph-mon[103445]: pgmap v264: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 2982/96750 objects degraded (3.082%); 2.7 MiB/s, 42 objects/s recovering 2024-04-05T03:03:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:30 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:30 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:30 smithi150 ceph-mon[107701]: pgmap v264: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 2982/96750 objects degraded (3.082%); 2.7 MiB/s, 42 objects/s recovering 2024-04-05T03:03:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:30 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:30 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:30.756 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:30 smithi196 ceph-mon[91352]: pgmap v264: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 2982/96750 objects degraded (3.082%); 2.7 MiB/s, 42 objects/s recovering 2024-04-05T03:03:30.756 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:30 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:30.756 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:30 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:31.792 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:31 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/791639030' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:31.888 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:31 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/791639030' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:31.888 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:03:31] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:03:31.888 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:31 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/791639030' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:32.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:32 smithi196 ceph-mon[91352]: pgmap v265: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 2982/96750 objects degraded (3.082%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:03:32.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 2982/96750 objects degraded (3.082%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:32 smithi150 ceph-mon[107701]: pgmap v265: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 2982/96750 objects degraded (3.082%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:03:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 2982/96750 objects degraded (3.082%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:32.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:32 smithi150 ceph-mon[103445]: pgmap v265: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 2982/96750 objects degraded (3.082%); 2.6 MiB/s, 41 objects/s recovering 2024-04-05T03:03:32.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 2982/96750 objects degraded (3.082%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:33.866 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:33 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:33.867 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:33 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:33.867 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:33 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:33.867 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:33 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:33.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:33 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:33.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:33 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:34.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:33 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:03:33.866+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:34.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: pgmap v266: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 2784/87861 objects degraded (3.169%); 3.6 MiB/s, 58 objects/s recovering 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:34.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:34 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/470640681' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: pgmap v266: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 2784/87861 objects degraded (3.169%); 3.6 MiB/s, 58 objects/s recovering 2024-04-05T03:03:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/470640681' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: pgmap v266: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 2784/87861 objects degraded (3.169%); 3.6 MiB/s, 58 objects/s recovering 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:03:34.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:03:34.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:03:34.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:34.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:34 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/470640681' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:35 smithi196 ceph-mon[91352]: pgmap v267: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 2784/87861 objects degraded (3.169%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:03:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:35 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:35 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:35.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:35 smithi150 ceph-mon[107701]: pgmap v267: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 2784/87861 objects degraded (3.169%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:03:35.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:35 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:35.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:35 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:35.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:35 smithi150 ceph-mon[103445]: pgmap v267: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 2784/87861 objects degraded (3.169%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:03:35.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:35 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:35.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:35 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:03:36.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:36 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3102650351' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:36.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:36 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3102650351' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:36.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:36 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3102650351' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:37 smithi196 ceph-mon[91352]: pgmap v268: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2784/86613 objects degraded (3.214%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:03:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 2784/86613 objects degraded (3.214%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:37.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:37 smithi150 ceph-mon[107701]: pgmap v268: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2784/86613 objects degraded (3.214%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:03:37.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 2784/86613 objects degraded (3.214%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:37.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:37 smithi150 ceph-mon[103445]: pgmap v268: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 2784/86613 objects degraded (3.214%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:03:37.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 2784/86613 objects degraded (3.214%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:38 smithi196 ceph-mon[91352]: pgmap v269: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 2578/79941 objects degraded (3.225%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:38 smithi150 ceph-mon[103445]: pgmap v269: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 2578/79941 objects degraded (3.225%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:38.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:38 smithi150 ceph-mon[107701]: pgmap v269: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 713 op/s; 2578/79941 objects degraded (3.225%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:39 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/543481886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:39 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/543481886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:39 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/543481886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:40.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:40 smithi196 ceph-mon[91352]: pgmap v270: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2578/75861 objects degraded (3.398%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:40 smithi150 ceph-mon[107701]: pgmap v270: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2578/75861 objects degraded (3.398%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:40 smithi150 ceph-mon[103445]: pgmap v270: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2578/75861 objects degraded (3.398%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:41.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:41.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:03:41] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2024-04-05T03:03:41.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:42.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: pgmap v271: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2578/75861 objects degraded (3.398%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: osdmap e107: 8 total, 8 up, 8 in 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1313781708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 2578/75861 objects degraded (3.398%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: pgmap v271: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2578/75861 objects degraded (3.398%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: osdmap e107: 8 total, 8 up, 8 in 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: failed to encode map e107 with expected crc 2024-04-05T03:03:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1313781708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:42.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:42 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 2578/75861 objects degraded (3.398%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: pgmap v271: 89 pgs: 86 active+clean, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 2578/75861 objects degraded (3.398%); 2.5 MiB/s, 39 objects/s recovering 2024-04-05T03:03:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: failed to encode map e107 with expected crc 2024-04-05T03:03:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: osdmap e107: 8 total, 8 up, 8 in 2024-04-05T03:03:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: failed to encode map e107 with expected crc 2024-04-05T03:03:43.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: failed to encode map e107 with expected crc 2024-04-05T03:03:43.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: failed to encode map e107 with expected crc 2024-04-05T03:03:43.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1313781708' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:43.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:42 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 2578/75861 objects degraded (3.398%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[107701]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[107701]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[107701]: osdmap e108: 8 total, 8 up, 8 in 2024-04-05T03:03:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[107701]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[107701]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[107701]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[103445]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[103445]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[103445]: osdmap e108: 8 total, 8 up, 8 in 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[103445]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[103445]: failed to encode map e108 with expected crc 2024-04-05T03:03:43.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:43 smithi150 ceph-mon[103445]: failed to encode map e108 with expected crc 2024-04-05T03:03:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:43 smithi196 ceph-mon[91352]: failed to encode map e108 with expected crc 2024-04-05T03:03:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:43 smithi196 ceph-mon[91352]: failed to encode map e108 with expected crc 2024-04-05T03:03:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:43 smithi196 ceph-mon[91352]: osdmap e108: 8 total, 8 up, 8 in 2024-04-05T03:03:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:43 smithi196 ceph-mon[91352]: failed to encode map e108 with expected crc 2024-04-05T03:03:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:43 smithi196 ceph-mon[91352]: failed to encode map e108 with expected crc 2024-04-05T03:03:44.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:43 smithi196 ceph-mon[91352]: failed to encode map e108 with expected crc 2024-04-05T03:03:44.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:44 smithi150 ceph-mon[107701]: pgmap v274: 89 pgs: 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 2378/67572 objects degraded (3.519%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:03:44.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:44 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2174834364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:44.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:44 smithi150 ceph-mon[103445]: pgmap v274: 89 pgs: 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 2378/67572 objects degraded (3.519%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:03:44.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:44 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2174834364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:45.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:44 smithi196 ceph-mon[91352]: pgmap v274: 89 pgs: 86 active+clean, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 2378/67572 objects degraded (3.519%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:03:45.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:44 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2174834364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:46.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:46 smithi150 ceph-mon[107701]: pgmap v275: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2182/65544 objects degraded (3.329%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:03:46.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:46 smithi150 ceph-mon[103445]: pgmap v275: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2182/65544 objects degraded (3.329%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:03:47.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:46 smithi196 ceph-mon[91352]: pgmap v275: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 2182/65544 objects degraded (3.329%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:03:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:47 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/372923144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 2182/65544 objects degraded (3.329%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:47 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/372923144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:47.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 2182/65544 objects degraded (3.329%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:47 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/372923144' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 2182/65544 objects degraded (3.329%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:48.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:48 smithi150 ceph-mon[107701]: pgmap v276: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2182/60174 objects degraded (3.626%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:03:48.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:48 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:03:48.883+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:03:48.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:48 smithi150 ceph-mon[103445]: pgmap v276: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2182/60174 objects degraded (3.626%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:03:49.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:48 smithi196 ceph-mon[91352]: pgmap v276: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 1.2 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2182/60174 objects degraded (3.626%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:03:49.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:49.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:49.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:03:49.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3969052824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:03:49.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:49 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3969052824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:50.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:49 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:50.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:49 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:03:50.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:49 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:03:50.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:49 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3969052824' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:50.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:50 smithi150 ceph-mon[103445]: pgmap v277: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 2019/55308 objects degraded (3.650%); 4.4 MiB/s, 69 objects/s recovering 2024-04-05T03:03:50.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:50 smithi150 ceph-mon[107701]: pgmap v277: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 2019/55308 objects degraded (3.650%); 4.4 MiB/s, 69 objects/s recovering 2024-04-05T03:03:51.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:50 smithi196 ceph-mon[91352]: pgmap v277: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 2019/55308 objects degraded (3.650%); 4.4 MiB/s, 69 objects/s recovering 2024-04-05T03:03:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:03:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:03:51] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2024-04-05T03:03:52.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:52 smithi150 ceph-mon[107701]: pgmap v278: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 2019/55308 objects degraded (3.650%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:03:52.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2192903200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:52.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 2019/55308 objects degraded (3.650%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:52.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:52 smithi150 ceph-mon[103445]: pgmap v278: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 2019/55308 objects degraded (3.650%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:03:52.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2192903200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:52.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 2019/55308 objects degraded (3.650%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:53.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:52 smithi196 ceph-mon[91352]: pgmap v278: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 2019/55308 objects degraded (3.650%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:03:53.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2192903200' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:53.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 2019/55308 objects degraded (3.650%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:55.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:54 smithi196 ceph-mon[91352]: pgmap v279: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 961 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 1845/46197 objects degraded (3.994%); 4.2 MiB/s, 66 objects/s recovering 2024-04-05T03:03:55.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:54 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1599742062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:55.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:54 smithi150 ceph-mon[103445]: pgmap v279: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 961 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 1845/46197 objects degraded (3.994%); 4.2 MiB/s, 66 objects/s recovering 2024-04-05T03:03:55.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:54 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1599742062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:55.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:54 smithi150 ceph-mon[107701]: pgmap v279: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 961 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 1845/46197 objects degraded (3.994%); 4.2 MiB/s, 66 objects/s recovering 2024-04-05T03:03:55.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:54 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1599742062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:56.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:56.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:03:57.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:56 smithi196 ceph-mon[91352]: pgmap v280: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 917 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 1845/44082 objects degraded (4.185%); 2.8 MiB/s, 44 objects/s recovering 2024-04-05T03:03:57.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:56 smithi150 ceph-mon[103445]: pgmap v280: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 917 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 1845/44082 objects degraded (4.185%); 2.8 MiB/s, 44 objects/s recovering 2024-04-05T03:03:57.301 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:56 smithi150 ceph-mon[107701]: pgmap v280: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 917 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 1845/44082 objects degraded (4.185%); 2.8 MiB/s, 44 objects/s recovering 2024-04-05T03:03:58.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 1845/44082 objects degraded (4.185%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:58.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:57 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1884272598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:58.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 1845/44082 objects degraded (4.185%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:58.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:57 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1884272598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 1845/44082 objects degraded (4.185%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:03:58.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:57 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1884272598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:03:59.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:03:58 smithi196 ceph-mon[91352]: pgmap v281: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 787 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1845/37833 objects degraded (4.877%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:03:59.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:03:58 smithi150 ceph-mon[107701]: pgmap v281: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 787 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1845/37833 objects degraded (4.877%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:03:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:03:58 smithi150 ceph-mon[103445]: pgmap v281: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 787 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1845/37833 objects degraded (4.877%); 1.8 MiB/s, 28 objects/s recovering 2024-04-05T03:04:01.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:00 smithi196 ceph-mon[91352]: pgmap v282: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 688 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1664/33051 objects degraded (5.035%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:04:01.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:00 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2382858349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:00 smithi150 ceph-mon[103445]: pgmap v282: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 688 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1664/33051 objects degraded (5.035%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:04:01.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:00 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2382858349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:00 smithi150 ceph-mon[107701]: pgmap v282: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 688 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 753 op/s; 1664/33051 objects degraded (5.035%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:04:01.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:00 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2382858349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:04:01] "GET /metrics HTTP/1.1" 200 36494 "" "Prometheus/2.43.0" 2024-04-05T03:04:03.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:02 smithi196 ceph-mon[91352]: pgmap v283: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 688 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1664/33051 objects degraded (5.035%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:04:03.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 1664/33051 objects degraded (5.035%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:03.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:02 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2629617349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:02 smithi150 ceph-mon[103445]: pgmap v283: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 688 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1664/33051 objects degraded (5.035%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:04:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 1664/33051 objects degraded (5.035%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:03.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:02 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2629617349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:03.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:02 smithi150 ceph-mon[107701]: pgmap v283: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 688 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 1664/33051 objects degraded (5.035%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:04:03.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 1664/33051 objects degraded (5.035%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:03.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:02 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2629617349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:04.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:03 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:04:03.904+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:04:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:04 smithi196 ceph-mon[91352]: pgmap v284: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 496 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 1477/23853 objects degraded (6.192%); 2.8 MiB/s, 45 objects/s recovering 2024-04-05T03:04:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:05.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:04 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:05.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:04 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:04:05.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[103445]: pgmap v284: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 496 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 1477/23853 objects degraded (6.192%); 2.8 MiB/s, 45 objects/s recovering 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[107701]: pgmap v284: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 496 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 873 op/s; 1477/23853 objects degraded (6.192%); 2.8 MiB/s, 45 objects/s recovering 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:05.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:04 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:04:06.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4162348536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:06.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4162348536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:06.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4162348536' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:07.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:06 smithi196 ceph-mon[91352]: pgmap v285: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 455 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1477/21885 objects degraded (6.749%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:04:07.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:06 smithi150 ceph-mon[103445]: pgmap v285: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 455 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1477/21885 objects degraded (6.749%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:04:07.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:06 smithi150 ceph-mon[107701]: pgmap v285: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 455 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 1477/21885 objects degraded (6.749%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:04:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 1477/21885 objects degraded (6.749%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:07 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3863077370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:08.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 1477/21885 objects degraded (6.749%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:08.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:07 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3863077370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:08.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 1477/21885 objects degraded (6.749%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:08.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:07 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3863077370' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:09.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:08 smithi196 ceph-mon[91352]: pgmap v286: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 1477/15549 objects degraded (9.499%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:04:09.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:08 smithi150 ceph-mon[103445]: pgmap v286: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 1477/15549 objects degraded (9.499%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:04:09.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:08 smithi150 ceph-mon[107701]: pgmap v286: 89 pgs: 87 active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped; 323 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 1477/15549 objects degraded (9.499%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:04:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:10 smithi196 ceph-mon[91352]: pgmap v287: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 221 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 1275/10662 objects degraded (11.958%); 3.0 MiB/s, 47 objects/s recovering 2024-04-05T03:04:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1649810815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:10 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:11.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:10 smithi150 ceph-mon[103445]: pgmap v287: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 221 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 1275/10662 objects degraded (11.958%); 3.0 MiB/s, 47 objects/s recovering 2024-04-05T03:04:11.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1649810815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:11.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:10 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:11.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:10 smithi150 ceph-mon[107701]: pgmap v287: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 221 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 754 op/s; 1275/10662 objects degraded (11.958%); 3.0 MiB/s, 47 objects/s recovering 2024-04-05T03:04:11.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1649810815' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:11.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:10 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:11.851 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:04:11] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2024-04-05T03:04:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:11 smithi196 ceph-mon[91352]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:11 smithi196 ceph-mon[91352]: osdmap e109: 8 total, 8 up, 8 in 2024-04-05T03:04:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:11 smithi196 ceph-mon[91352]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:11 smithi196 ceph-mon[91352]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:11 smithi196 ceph-mon[91352]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[107701]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[107701]: osdmap e109: 8 total, 8 up, 8 in 2024-04-05T03:04:12.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[107701]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[107701]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[107701]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[103445]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[103445]: osdmap e109: 8 total, 8 up, 8 in 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[103445]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[103445]: failed to encode map e109 with expected crc 2024-04-05T03:04:12.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:11 smithi150 ceph-mon[103445]: failed to encode map e109 with expected crc 2024-04-05T03:04:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: pgmap v289: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 221 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 1275/10662 objects degraded (11.958%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:04:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: osdmap e110: 8 total, 8 up, 8 in 2024-04-05T03:04:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 1275/10662 objects degraded (11.958%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:13.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:12 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4220755700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: pgmap v289: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 221 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 1275/10662 objects degraded (11.958%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:04:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: osdmap e110: 8 total, 8 up, 8 in 2024-04-05T03:04:13.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 1275/10662 objects degraded (11.958%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4220755700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: pgmap v289: 89 pgs: 87 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped; 221 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 1275/10662 objects degraded (11.958%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: osdmap e110: 8 total, 8 up, 8 in 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: failed to encode map e110 with expected crc 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 1275/10662 objects degraded (11.958%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-04-05T03:04:13.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:12 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4220755700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:13.238 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Removed 124534 objects 2024-04-05T03:04:13.238 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Clean up completed and total clean up time :175.498 2024-04-05T03:04:15.128 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-04-05T03:04:15.128 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-04-05T03:04:15.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:14 smithi196 ceph-mon[91352]: pgmap v291: 89 pgs: 87 active+clean, 2 active+recovering+undersized+degraded+remapped; 70 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1098/3420 objects degraded (32.105%); 2.2 MiB/s, 47 objects/s recovering 2024-04-05T03:04:15.154 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:14 smithi150 ceph-mon[103445]: pgmap v291: 89 pgs: 87 active+clean, 2 active+recovering+undersized+degraded+remapped; 70 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1098/3420 objects degraded (32.105%); 2.2 MiB/s, 47 objects/s recovering 2024-04-05T03:04:15.155 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:14 smithi150 ceph-mon[107701]: pgmap v291: 89 pgs: 87 active+clean, 2 active+recovering+undersized+degraded+remapped; 70 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 1098/3420 objects degraded (32.105%); 2.2 MiB/s, 47 objects/s recovering 2024-04-05T03:04:15.397 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:16.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:15 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3652626896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:16.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:15 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3652626896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:15 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3652626896' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:16 smithi150 ceph-mon[103445]: pgmap v292: 89 pgs: 88 active+clean, 1 active+recovering+undersized+degraded+remapped; 31 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 980/1536 objects degraded (63.802%); 3.1 MiB/s, 62 objects/s recovering 2024-04-05T03:04:16.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:16 smithi150 ceph-mon[107701]: pgmap v292: 89 pgs: 88 active+clean, 1 active+recovering+undersized+degraded+remapped; 31 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 980/1536 objects degraded (63.802%); 3.1 MiB/s, 62 objects/s recovering 2024-04-05T03:04:17.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:16 smithi196 ceph-mon[91352]: pgmap v292: 89 pgs: 88 active+clean, 1 active+recovering+undersized+degraded+remapped; 31 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 583 op/s; 980/1536 objects degraded (63.802%); 3.1 MiB/s, 62 objects/s recovering 2024-04-05T03:04:17.874 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_2' removed 2024-04-05T03:04:18.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: osdmap e111: 8 total, 8 up, 8 in 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 980/1536 objects degraded (63.802%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/109064918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:18.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3640908443' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:04:18.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: osdmap e111: 8 total, 8 up, 8 in 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 980/1536 objects degraded (63.802%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/109064918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3640908443' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: osdmap e111: 8 total, 8 up, 8 in 2024-04-05T03:04:18.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 980/1536 objects degraded (63.802%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-04-05T03:04:18.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: failed to encode map e111 with expected crc 2024-04-05T03:04:18.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/109064918' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:18.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3640908443' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: pgmap v294: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 1.8 MiB/s, 187 objects/s recovering 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3640908443' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: osdmap e112: 8 total, 8 up, 8 in 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: failed to encode map e111 with expected crc 2024-04-05T03:04:19.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:18 smithi196 ceph-mon[91352]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: pgmap v294: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 1.8 MiB/s, 187 objects/s recovering 2024-04-05T03:04:19.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3640908443' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: osdmap e112: 8 total, 8 up, 8 in 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: failed to encode map e111 with expected crc 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[103445]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: pgmap v294: 89 pgs: 1 active+undersized+remapped, 88 active+clean; 3.3 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 1.8 MiB/s, 187 objects/s recovering 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3640908443' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: osdmap e112: 8 total, 8 up, 8 in 2024-04-05T03:04:19.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: failed to encode map e111 with expected crc 2024-04-05T03:04:19.229 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:18 smithi150 ceph-mon[107701]: failed to encode map e112 with expected crc 2024-04-05T03:04:19.385 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-04-05T03:04:19.386 INFO:tasks.radosbench:Beginning radosbench... 2024-04-05T03:04:19.386 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-04-05T03:04:19.386 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool create unique_pool_3 16 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[107701]: Upgrade: osd.1 is safe to restart 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2748605673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[103445]: Upgrade: osd.1 is safe to restart 2024-04-05T03:04:19.908 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:19 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2748605673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:20.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:19 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:20.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:19 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-05T03:04:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:19 smithi196 ceph-mon[91352]: Upgrade: osd.1 is safe to restart 2024-04-05T03:04:20.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:19 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2748605673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:20.163 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:21.091 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[103445]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:21.091 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[103445]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-04-05T03:04:21.091 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:21.091 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:04:21.091 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:21.092 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[107701]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:21.092 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[107701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-04-05T03:04:21.092 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:21.092 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:04:21.092 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:20 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:21.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:20 smithi196 ceph-mon[91352]: pgmap v296: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:21.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:20 smithi196 ceph-mon[91352]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1 pg undersized) 2024-04-05T03:04:21.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:20 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:21.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:20 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-05T03:04:21.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:20 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:04:21] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2024-04-05T03:04:22.079 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:21 smithi150 ceph-mon[107701]: Upgrade: Updating osd.1 2024-04-05T03:04:22.079 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:21 smithi150 ceph-mon[107701]: Deploying daemon osd.1 on smithi150 2024-04-05T03:04:22.079 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:21 smithi150 ceph-mon[103445]: Upgrade: Updating osd.1 2024-04-05T03:04:22.079 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:21 smithi150 ceph-mon[103445]: Deploying daemon osd.1 on smithi150 2024-04-05T03:04:22.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:21 smithi196 ceph-mon[91352]: Upgrade: Updating osd.1 2024-04-05T03:04:22.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:21 smithi196 ceph-mon[91352]: Deploying daemon osd.1 on smithi150 2024-04-05T03:04:22.906 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_3' created 2024-04-05T03:04:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:22 smithi196 ceph-mon[91352]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1428461844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2121980890' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-05T03:04:23.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:22 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-05T03:04:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[103445]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1428461844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2121980890' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-05T03:04:23.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-05T03:04:23.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[107701]: pgmap v297: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1428461844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2121980890' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-05T03:04:23.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:22 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2024-04-05T03:04:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-05T03:04:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[107701]: failed to encode map e113 with expected crc 2024-04-05T03:04:24.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[107701]: failed to encode map e113 with expected crc 2024-04-05T03:04:24.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[107701]: osdmap e113: 8 total, 8 up, 8 in 2024-04-05T03:04:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-05T03:04:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[103445]: failed to encode map e113 with expected crc 2024-04-05T03:04:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[103445]: failed to encode map e113 with expected crc 2024-04-05T03:04:24.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:23 smithi150 ceph-mon[103445]: osdmap e113: 8 total, 8 up, 8 in 2024-04-05T03:04:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:23 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-04-05T03:04:24.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:23 smithi196 ceph-mon[91352]: failed to encode map e113 with expected crc 2024-04-05T03:04:24.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:23 smithi196 ceph-mon[91352]: failed to encode map e113 with expected crc 2024-04-05T03:04:24.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:23 smithi196 ceph-mon[91352]: osdmap e113: 8 total, 8 up, 8 in 2024-04-05T03:04:24.703 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-04-05T03:04:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: pgmap v299: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:04:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: osdmap e114: 8 total, 8 up, 8 in 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3906033878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: pgmap v299: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:04:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: osdmap e114: 8 total, 8 up, 8 in 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: failed to encode map e114 with expected crc 2024-04-05T03:04:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:24 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3906033878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:25.293 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:04:25.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: pgmap v299: 89 pgs: 3 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: osdmap e114: 8 total, 8 up, 8 in 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: failed to encode map e114 with expected crc 2024-04-05T03:04:25.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:24 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3906033878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:25.727 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:25 smithi150 systemd[1]: Stopping Ceph osd.1 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[103445]: failed to encode map e115 with expected crc 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[103445]: failed to encode map e115 with expected crc 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[103445]: osdmap e115: 8 total, 8 up, 8 in 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[107701]: failed to encode map e115 with expected crc 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[107701]: failed to encode map e115 with expected crc 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[107701]: osdmap e115: 8 total, 8 up, 8 in 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:26.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:25 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:25 smithi196 ceph-mon[91352]: failed to encode map e115 with expected crc 2024-04-05T03:04:26.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:25 smithi196 ceph-mon[91352]: failed to encode map e115 with expected crc 2024-04-05T03:04:26.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:25 smithi196 ceph-mon[91352]: osdmap e115: 8 total, 8 up, 8 in 2024-04-05T03:04:26.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:25 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:26.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:25 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:26.917 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[52666]: 2024-04-05T03:04:26.706+0000 7faf16d4d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:04:26.917 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[52666]: 2024-04-05T03:04:26.706+0000 7faf16d4d700 -1 osd.1 115 *** Got signal Terminated *** 2024-04-05T03:04:26.917 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[52666]: 2024-04-05T03:04:26.706+0000 7faf16d4d700 -1 osd.1 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:04:27.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[103445]: pgmap v302: 89 pgs: 3 creating+peering, 10 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:27.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:27.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[103445]: osd.1 marked itself down and dead 2024-04-05T03:04:27.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2421238138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[107701]: pgmap v302: 89 pgs: 3 creating+peering, 10 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[107701]: osd.1 marked itself down and dead 2024-04-05T03:04:27.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:26 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2421238138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:27.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:26 smithi196 ceph-mon[91352]: pgmap v302: 89 pgs: 3 creating+peering, 10 unknown, 76 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:27.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:26 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:27.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:26 smithi196 ceph-mon[91352]: osd.1 marked itself down and dead 2024-04-05T03:04:27.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:26 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2421238138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:27.683 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:27 smithi150 podman[125912]: 2024-04-05 03:04:27.379956515 +0000 UTC m=+1.225631181 container died 2c503b0a41b4be741f1de6139115f2689388f6e433205469d409bc0ffbe1eb78 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:04:27.968 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:27 smithi150 podman[125912]: 2024-04-05 03:04:27.8661887 +0000 UTC m=+1.711863366 container cleanup 2c503b0a41b4be741f1de6139115f2689388f6e433205469d409bc0ffbe1eb78 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.build-date=20240403, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS) 2024-04-05T03:04:27.968 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:27 smithi150 bash[125912]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1 2024-04-05T03:04:27.989 INFO:teuthology.orchestra.run.smithi150.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-04-05T03:04:28.230 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:28 smithi150 podman[126028]: 2024-04-05 03:04:28.160141071 +0000 UTC m=+0.781076850 container remove 2c503b0a41b4be741f1de6139115f2689388f6e433205469d409bc0ffbe1eb78 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-05T03:04:28.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[103445]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:04:28.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[103445]: failed to encode map e116 with expected crc 2024-04-05T03:04:28.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[103445]: osdmap e116: 8 total, 7 up, 8 in 2024-04-05T03:04:28.230 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[103445]: failed to encode map e116 with expected crc 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/600485384' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[107701]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[107701]: failed to encode map e116 with expected crc 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[107701]: osdmap e116: 8 total, 7 up, 8 in 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[107701]: failed to encode map e116 with expected crc 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/600485384' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:04:28.231 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:27 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:04:28.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:27 smithi196 ceph-mon[91352]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:04:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:27 smithi196 ceph-mon[91352]: failed to encode map e116 with expected crc 2024-04-05T03:04:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:27 smithi196 ceph-mon[91352]: osdmap e116: 8 total, 7 up, 8 in 2024-04-05T03:04:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:27 smithi196 ceph-mon[91352]: failed to encode map e116 with expected crc 2024-04-05T03:04:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:27 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/600485384' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:04:28.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:27 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-04-05T03:04:29.097 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:29 smithi150 podman[126172]: 2024-04-05 03:04:28.956120493 +0000 UTC m=+0.022540920 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:04:29.098 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:29 smithi150 podman[126172]: 2024-04-05 03:04:29.098660703 +0000 UTC m=+0.165081106 container create f9268231c8d455a901c8a2462dd3e267876e5e66fe636d86ff9931e9e644cc52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, RELEASE=squid-0028525, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS) 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: pgmap v304: 89 pgs: 4 peering, 6 stale+active+clean, 3 creating+peering, 3 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: osdmap e117: 8 total, 7 up, 8 in 2024-04-05T03:04:29.098 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[103445]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: pgmap v304: 89 pgs: 4 peering, 6 stale+active+clean, 3 creating+peering, 3 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: osdmap e117: 8 total, 7 up, 8 in 2024-04-05T03:04:29.099 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:28 smithi150 ceph-mon[107701]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: pgmap v304: 89 pgs: 4 peering, 6 stale+active+clean, 3 creating+peering, 3 unknown, 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: osdmap e117: 8 total, 7 up, 8 in 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:28 smithi196 ceph-mon[91352]: failed to encode map e117 with expected crc 2024-04-05T03:04:29.417 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:29 smithi150 podman[126172]: 2024-04-05 03:04:29.335271649 +0000 UTC m=+0.401692067 container init f9268231c8d455a901c8a2462dd3e267876e5e66fe636d86ff9931e9e644cc52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=squid-0028525) 2024-04-05T03:04:29.417 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:29 smithi150 podman[126172]: 2024-04-05 03:04:29.341268971 +0000 UTC m=+0.407689377 container start f9268231c8d455a901c8a2462dd3e267876e5e66fe636d86ff9931e9e644cc52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403) 2024-04-05T03:04:29.417 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:29 smithi150 podman[126172]: 2024-04-05 03:04:29.418123126 +0000 UTC m=+0.484543542 container attach f9268231c8d455a901c8a2462dd3e267876e5e66fe636d86ff9931e9e644cc52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:04:29.727 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:29 smithi150 podman[126172]: 2024-04-05 03:04:29.626045759 +0000 UTC m=+0.692466163 container died f9268231c8d455a901c8a2462dd3e267876e5e66fe636d86ff9931e9e644cc52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, RELEASE=squid-0028525, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS) 2024-04-05T03:04:29.831 DEBUG:teuthology.orchestra.run.smithi150:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-04-05T03:04:29.834 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-04-05T03:04:29.834 INFO:teuthology.orchestra.run:waiting for 3000 2024-04-05T03:04:29.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:hints = 1 2024-04-05T03:04:29.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-04-05T03:04:29.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object prefix: benchmark_data_smithi150_126233 2024-04-05T03:04:29.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:04:29.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 0 0 0 0 0 0 - 0 2024-04-05T03:04:30.287 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/592720865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:30.397 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/592720865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:30.398 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/592720865' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:30.727 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:30 smithi150 podman[126221]: 2024-04-05 03:04:30.398648297 +0000 UTC m=+0.766948436 container remove f9268231c8d455a901c8a2462dd3e267876e5e66fe636d86ff9931e9e644cc52 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-04-05T03:04:30.728 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:30 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1.service: Deactivated successfully. 2024-04-05T03:04:30.728 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:30 smithi150 systemd[1]: Stopped Ceph osd.1 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:04:30.728 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:30 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1.service: Consumed 16min 54.598s CPU time. 2024-04-05T03:04:30.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 1 16 4502 4486 280.358 280.375 0.00481292 0.00355516 2024-04-05T03:04:31.060 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:30 smithi150 systemd[1]: Starting Ceph osd.1 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:04:31.060 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:30 smithi150 ceph-mon[103445]: pgmap v306: 89 pgs: 1 active+undersized+degraded, 13 active+undersized, 4 peering, 3 stale+active+clean, 1 unknown, 67 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-05T03:04:31.060 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:30 smithi150 ceph-mon[103445]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:04:31.061 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:30 smithi150 ceph-mon[103445]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:04:31.061 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:30 smithi150 ceph-mon[107701]: pgmap v306: 89 pgs: 1 active+undersized+degraded, 13 active+undersized, 4 peering, 3 stale+active+clean, 1 unknown, 67 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-05T03:04:31.061 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:30 smithi150 ceph-mon[107701]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:04:31.061 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:30 smithi150 ceph-mon[107701]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:04:31.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:30 smithi196 ceph-mon[91352]: pgmap v306: 89 pgs: 1 active+undersized+degraded, 13 active+undersized, 4 peering, 3 stale+active+clean, 1 unknown, 67 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-05T03:04:31.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:30 smithi196 ceph-mon[91352]: Health check failed: Degraded data redundancy: 1/72 objects degraded (1.389%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:04:31.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:30 smithi196 ceph-mon[91352]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-05T03:04:31.411 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:31 smithi150 podman[126355]: 2024-04-05 03:04:30.96756458 +0000 UTC m=+0.032764493 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:04:31.412 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:31 smithi150 podman[126355]: 2024-04-05 03:04:31.119676217 +0000 UTC m=+0.184876114 container create 56fa66b630a0bfbb21643284787a30910d8b45ebb93afd2d673e16f0db75079c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-05T03:04:31.412 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:31 smithi150 podman[126355]: 2024-04-05 03:04:31.412178535 +0000 UTC m=+0.477378446 container init 56fa66b630a0bfbb21643284787a30910d8b45ebb93afd2d673e16f0db75079c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-0028525, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-05T03:04:31.727 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:31 smithi150 podman[126355]: 2024-04-05 03:04:31.424071695 +0000 UTC m=+0.489271595 container start 56fa66b630a0bfbb21643284787a30910d8b45ebb93afd2d673e16f0db75079c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate, RELEASE=squid-0028525, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-05T03:04:31.727 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:31 smithi150 podman[126355]: 2024-04-05 03:04:31.520054094 +0000 UTC m=+0.585253995 container attach 56fa66b630a0bfbb21643284787a30910d8b45ebb93afd2d673e16f0db75079c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-05T03:04:31.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:04:31] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2024-04-05T03:04:31.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 2 16 8397 8381 261.886 243.438 0.00445064 0.00381071 2024-04-05T03:04:32.269 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:31 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2596685560' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:32.269 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:31 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2596685560' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-05T03:04:32.270 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 bash[126355]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-05T03:04:32.271 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate[126373]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-05T03:04:32.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:31 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2596685560' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:32.617 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:32 smithi150 podman[126529]: 2024-04-05 03:04:32.318742829 +0000 UTC m=+0.038586988 container died 56fa66b630a0bfbb21643284787a30910d8b45ebb93afd2d673e16f0db75079c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:04:32.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 3 16 12293 12277 255.75 243.5 0.00256572 0.00390428 2024-04-05T03:04:33.308 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-mon[103445]: pgmap v307: 89 pgs: 1 active+undersized+degraded, 13 active+undersized, 4 peering, 3 stale+active+clean, 1 unknown, 67 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-05T03:04:33.308 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:32 smithi150 ceph-mon[107701]: pgmap v307: 89 pgs: 1 active+undersized+degraded, 13 active+undersized, 4 peering, 3 stale+active+clean, 1 unknown, 67 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-05T03:04:33.309 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:33 smithi150 podman[126529]: 2024-04-05 03:04:33.008975368 +0000 UTC m=+0.728819517 container remove 56fa66b630a0bfbb21643284787a30910d8b45ebb93afd2d673e16f0db75079c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:04:33.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:32 smithi196 ceph-mon[91352]: pgmap v307: 89 pgs: 1 active+undersized+degraded, 13 active+undersized, 4 peering, 3 stale+active+clean, 1 unknown, 67 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 1/72 objects degraded (1.389%) 2024-04-05T03:04:33.613 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:33 smithi150 podman[126584]: 2024-04-05 03:04:33.214064933 +0000 UTC m=+0.036277463 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:04:33.613 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:33 smithi150 podman[126584]: 2024-04-05 03:04:33.343715056 +0000 UTC m=+0.165927574 container create 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-05T03:04:33.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 4 16 15820 15804 246.918 220.438 0.005523 0.00404218 2024-04-05T03:04:33.977 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:33 smithi150 podman[126584]: 2024-04-05 03:04:33.613690111 +0000 UTC m=+0.435902629 container init 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-05T03:04:33.977 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:33 smithi150 podman[126584]: 2024-04-05 03:04:33.624068815 +0000 UTC m=+0.446281332 container start 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-05T03:04:33.978 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:33 smithi150 ceph-osd[126619]: -- 172.21.15.150:0/2688940732 <== mon.2 v2:172.21.15.150:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e85b303680 con 0x55e85b2ccc00 2024-04-05T03:04:34.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:34 smithi196 ceph-mon[91352]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-05T03:04:34.477 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:34 smithi150 bash[126584]: 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 2024-04-05T03:04:34.477 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:34 smithi150 systemd[1]: Started Ceph osd.1 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:04:34.477 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:34 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[126597]: 2024-04-05T03:04:34.177+0000 7f07282a37c0 -1 Falling back to public interface 2024-04-05T03:04:34.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:34 smithi150 ceph-mon[107701]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-05T03:04:34.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:34 smithi150 ceph-mon[103445]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2024-04-05T03:04:34.881 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 5 16 18800 18784 234.78 186.25 0.00774851 0.00425416 2024-04-05T03:04:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: pgmap v308: 89 pgs: 9 active+undersized+degraded, 26 active+undersized, 54 active+clean; 465 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 4478/22353 objects degraded (20.033%) 2024-04-05T03:04:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1392991107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: pgmap v308: 89 pgs: 9 active+undersized+degraded, 26 active+undersized, 54 active+clean; 465 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 4478/22353 objects degraded (20.033%) 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1392991107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.485 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: pgmap v308: 89 pgs: 9 active+undersized+degraded, 26 active+undersized, 54 active+clean; 465 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 4478/22353 objects degraded (20.033%) 2024-04-05T03:04:35.485 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.485 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.485 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:35.485 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1392991107' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:35.485 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.486 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:35.882 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 6 16 21723 21707 226.069 182.688 0.00301423 0.00441813 2024-04-05T03:04:36.882 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 7 16 24300 24284 216.781 161.062 0.0070307 0.00460791 2024-04-05T03:04:37.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:36 smithi196 ceph-mon[91352]: pgmap v309: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 725 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 5524/34863 objects degraded (15.845%) 2024-04-05T03:04:37.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:36 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:37.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:36 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 5524/34863 objects degraded (15.845%), 10 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:37.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:36 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2011488631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:37.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[107701]: pgmap v309: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 725 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 5524/34863 objects degraded (15.845%) 2024-04-05T03:04:37.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:37.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 5524/34863 objects degraded (15.845%), 10 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:37.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2011488631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:37.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[103445]: pgmap v309: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 725 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s; 5524/34863 objects degraded (15.845%) 2024-04-05T03:04:37.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:37.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 5524/34863 objects degraded (15.845%), 10 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:37.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:36 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2011488631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:37.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 8 16 26561 26545 207.348 141.312 0.00437557 0.00481752 2024-04-05T03:04:38.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 9 16 29092 29076 201.884 158.188 0.00543852 0.00494799 2024-04-05T03:04:39.279 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:38 smithi196 ceph-mon[91352]: pgmap v310: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 7402/58701 objects degraded (12.610%) 2024-04-05T03:04:39.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:38 smithi150 ceph-mon[107701]: pgmap v310: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 7402/58701 objects degraded (12.610%) 2024-04-05T03:04:39.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:38 smithi150 ceph-mon[103445]: pgmap v310: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s; 7402/58701 objects degraded (12.610%) 2024-04-05T03:04:39.882 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 10 16 31412 31396 196.195 145 0.00741737 0.00509161 2024-04-05T03:04:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3366928240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:40.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3366928240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:40.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:40.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3366928240' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:40.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 11 16 33401 33385 189.66 124.312 0.00542755 0.00526781 2024-04-05T03:04:41.378 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-mon[103445]: pgmap v311: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s; 11779/79113 objects degraded (14.889%) 2024-04-05T03:04:41.378 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:41.378 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:41.379 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-mon[107701]: pgmap v311: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s; 11779/79113 objects degraded (14.889%) 2024-04-05T03:04:41.379 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:41.379 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:41.379 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[126597]: 2024-04-05T03:04:41.169+0000 7f07282a37c0 -1 osd.1 115 log_to_monitors true 2024-04-05T03:04:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:41 smithi196 ceph-mon[91352]: pgmap v311: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s; 11779/79113 objects degraded (14.889%) 2024-04-05T03:04:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:41 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:41.729 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:04:41] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2024-04-05T03:04:41.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 12 16 35491 35475 184.739 130.625 0.00632685 0.00540566 2024-04-05T03:04:42.130 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:42 smithi150 ceph-mon[103445]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:04:42.130 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:42 smithi150 ceph-mon[107701]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:04:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:42 smithi196 ceph-mon[91352]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-05T03:04:42.843 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:04:42 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[126597]: 2024-04-05T03:04:42.526+0000 7f07156f3700 -1 osd.1 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:04:42.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 13 16 36816 36800 176.899 82.8125 0.00986166 0.00564765 2024-04-05T03:04:43.213 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: pgmap v312: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s; 11779/79113 objects degraded (14.889%) 2024-04-05T03:04:43.213 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T03:04:43.213 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: failed to encode map e118 with expected crc 2024-04-05T03:04:43.213 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: failed to encode map e118 with expected crc 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: osdmap e118: 8 total, 7 up, 8 in 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 11779/79113 objects degraded (14.889%), 10 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2749987762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[103445]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: pgmap v312: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s; 11779/79113 objects degraded (14.889%) 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: failed to encode map e118 with expected crc 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: failed to encode map e118 with expected crc 2024-04-05T03:04:43.214 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: osdmap e118: 8 total, 7 up, 8 in 2024-04-05T03:04:43.215 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:04:43.215 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 11779/79113 objects degraded (14.889%), 10 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:43.215 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2749987762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:43.215 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:43 smithi150 ceph-mon[107701]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' 2024-04-05T03:04:43.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: pgmap v312: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 1.6 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s; 11779/79113 objects degraded (14.889%) 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: failed to encode map e118 with expected crc 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: failed to encode map e118 with expected crc 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: osdmap e118: 8 total, 7 up, 8 in 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 11779/79113 objects degraded (14.889%), 10 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2749987762' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:43.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:43 smithi196 ceph-mon[91352]: from='osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467]' entity='osd.1' 2024-04-05T03:04:43.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 14 16 38014 37998 169.61 74.875 0.00298797 0.00574853 2024-04-05T03:04:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:44 smithi196 ceph-mon[91352]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:04:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:44 smithi196 ceph-mon[91352]: failed to encode map e119 with expected crc 2024-04-05T03:04:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:44 smithi196 ceph-mon[91352]: osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467] boot 2024-04-05T03:04:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:44 smithi196 ceph-mon[91352]: osdmap e119: 8 total, 8 up, 8 in 2024-04-05T03:04:44.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:44 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:04:44.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[107701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:04:44.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[107701]: failed to encode map e119 with expected crc 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[107701]: osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467] boot 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[107701]: osdmap e119: 8 total, 8 up, 8 in 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[103445]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[103445]: failed to encode map e119 with expected crc 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[103445]: osd.1 [v2:172.21.15.150:6810/3505931467,v1:172.21.15.150:6811/3505931467] boot 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[103445]: osdmap e119: 8 total, 8 up, 8 in 2024-04-05T03:04:44.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:44 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-05T03:04:44.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 15 16 38014 37998 158.303 0 - 0.00574853 2024-04-05T03:04:45.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e119 with expected crc 2024-04-05T03:04:45.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: pgmap v315: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 179 MiB/s wr, 2.86k op/s; 15764/103491 objects degraded (15.232%) 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4184112446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: osdmap e120: 8 total, 8 up, 8 in 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:45 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e119 with expected crc 2024-04-05T03:04:45.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: pgmap v315: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 179 MiB/s wr, 2.86k op/s; 15764/103491 objects degraded (15.232%) 2024-04-05T03:04:45.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4184112446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:45.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: osdmap e120: 8 total, 8 up, 8 in 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e119 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: pgmap v315: 89 pgs: 10 active+undersized+degraded, 25 active+undersized, 54 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 179 MiB/s wr, 2.86k op/s; 15764/103491 objects degraded (15.232%) 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4184112446' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: osdmap e120: 8 total, 8 up, 8 in 2024-04-05T03:04:45.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:45 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:45.883 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 16 16 38198 38182 149.127 5.75 0.00300264 0.00669907 2024-04-05T03:04:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: osdmap e121: 8 total, 8 up, 8 in 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.388 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:46 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: osdmap e121: 8 total, 8 up, 8 in 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.479 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e120 with expected crc 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: osdmap e121: 8 total, 8 up, 8 in 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.480 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:46 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:46.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 17 16 39371 39355 144.663 73.3125 0.0125009 0.00690509 2024-04-05T03:04:47.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:47 smithi196 ceph-mon[91352]: pgmap v318: 89 pgs: 3 peering, 9 active+undersized+degraded, 23 active+undersized, 54 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15763/107871 objects degraded (14.613%) 2024-04-05T03:04:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:47 smithi196 ceph-mon[91352]: failed to encode map e121 with expected crc 2024-04-05T03:04:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:47 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2687840843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 15763/107871 objects degraded (14.613%), 9 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[107701]: pgmap v318: 89 pgs: 3 peering, 9 active+undersized+degraded, 23 active+undersized, 54 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15763/107871 objects degraded (14.613%) 2024-04-05T03:04:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[107701]: failed to encode map e121 with expected crc 2024-04-05T03:04:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2687840843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:47.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 15763/107871 objects degraded (14.613%), 9 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[103445]: pgmap v318: 89 pgs: 3 peering, 9 active+undersized+degraded, 23 active+undersized, 54 active+clean; 2.2 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s; 15763/107871 objects degraded (14.613%) 2024-04-05T03:04:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[103445]: failed to encode map e121 with expected crc 2024-04-05T03:04:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2687840843' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:47.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 15763/107871 objects degraded (14.613%), 9 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:47.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 18 16 40605 40589 140.911 77.125 0.0205016 0.00709005 2024-04-05T03:04:48.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 19 16 41979 41963 138.014 85.875 0.0194139 0.00723897 2024-04-05T03:04:49.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:49 smithi196 ceph-mon[91352]: pgmap v319: 89 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 3 active+undersized, 76 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 16001/113760 objects degraded (14.066%); 669 KiB/s, 10 objects/s recovering 2024-04-05T03:04:49.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:49 smithi150 ceph-mon[107701]: pgmap v319: 89 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 3 active+undersized, 76 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 16001/113760 objects degraded (14.066%); 669 KiB/s, 10 objects/s recovering 2024-04-05T03:04:49.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:49 smithi150 ceph-mon[103445]: pgmap v319: 89 pgs: 3 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized+degraded, 3 active+undersized, 76 active+clean; 2.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 16001/113760 objects degraded (14.066%); 669 KiB/s, 10 objects/s recovering 2024-04-05T03:04:49.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:04:49.885803+0000 min lat: 0.000950756 max lat: 2.23538 avg lat: 0.00739847 2024-04-05T03:04:49.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:04:49.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 20 16 43239 43223 135.051 78.75 0.0270015 0.00739847 2024-04-05T03:04:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:50 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3915618242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:50 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3915618242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:50 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3915618242' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:50.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 21 16 44494 44478 132.355 78.4375 0.00877371 0.007549 2024-04-05T03:04:51.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:51 smithi150 ceph-mon[107701]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 16770/121497 objects degraded (13.803%); 645 KiB/s, 10 objects/s recovering 2024-04-05T03:04:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:51 smithi150 ceph-mon[103445]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 16770/121497 objects degraded (13.803%); 645 KiB/s, 10 objects/s recovering 2024-04-05T03:04:51.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:51 smithi196 ceph-mon[91352]: pgmap v320: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 16770/121497 objects degraded (13.803%); 645 KiB/s, 10 objects/s recovering 2024-04-05T03:04:51.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 22 16 45709 45693 129.79 75.9375 0.00546625 0.00769696 2024-04-05T03:04:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:04:51] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.43.0" 2024-04-05T03:04:52.449 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 16770/121497 objects degraded (13.803%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:52.449 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 16770/121497 objects degraded (13.803%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 16770/121497 objects degraded (13.803%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:52.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 23 16 46902 46886 127.389 74.5625 0.013706 0.0078438 2024-04-05T03:04:53.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:53 smithi150 ceph-mon[107701]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 16770/121497 objects degraded (13.803%); 496 KiB/s, 7 objects/s recovering 2024-04-05T03:04:53.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:53 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3970135324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:53.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:53 smithi150 ceph-mon[103445]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 16770/121497 objects degraded (13.803%); 496 KiB/s, 7 objects/s recovering 2024-04-05T03:04:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:53 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3970135324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:53 smithi196 ceph-mon[91352]: pgmap v321: 89 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 82 active+clean; 2.5 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 47 MiB/s wr, 750 op/s; 16770/121497 objects degraded (13.803%); 496 KiB/s, 7 objects/s recovering 2024-04-05T03:04:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:53 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3970135324' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:53.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 24 16 48150 48134 125.331 78 0.00705444 0.00797329 2024-04-05T03:04:53.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:04:53 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:04:53.521+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:04:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 16500/135876 objects degraded (12.143%); 2.4 MiB/s, 37 objects/s recovering 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 16500/135876 objects degraded (12.143%); 2.7 MiB/s, 42 objects/s recovering 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:04:54.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:54 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:04:54.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 16500/135876 objects degraded (12.143%); 2.4 MiB/s, 37 objects/s recovering 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 16500/135876 objects degraded (12.143%); 2.7 MiB/s, 42 objects/s recovering 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:04:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: pgmap v322: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 16500/135876 objects degraded (12.143%); 2.4 MiB/s, 37 objects/s recovering 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: pgmap v323: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.8 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s; 16500/135876 objects degraded (12.143%); 2.7 MiB/s, 42 objects/s recovering 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:04:54.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:04:54.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:04:54.730 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:54 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:04:54.884 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 25 15 49362 49347 123.35 75.8125 0.00464255 0.00809881 2024-04-05T03:04:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:55 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2354401371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:55 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2354401371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:55.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:55 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2354401371' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:55.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 26 16 50609 50593 121.6 77.875 0.00818952 0.00821859 2024-04-05T03:04:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:56 smithi196 ceph-mon[91352]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 16500/140340 objects degraded (11.757%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:04:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:56 smithi150 ceph-mon[107701]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 16500/140340 objects degraded (11.757%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:04:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:56 smithi150 ceph-mon[103445]: pgmap v324: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.9 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 16500/140340 objects degraded (11.757%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:04:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:04:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:04:56.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 27 16 51877 51861 120.032 79.25 0.00822029 0.00832397 2024-04-05T03:04:57.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 16500/140340 objects degraded (11.757%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:57.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 16500/140340 objects degraded (11.757%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:57.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 16500/140340 objects degraded (11.757%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:04:57.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 28 16 53052 53036 118.367 73.4375 0.00726306 0.00844238 2024-04-05T03:04:58.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:58 smithi196 ceph-mon[91352]: pgmap v325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 16221/148356 objects degraded (10.934%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:04:58.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:04:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2038386232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:58 smithi150 ceph-mon[107701]: pgmap v325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 16221/148356 objects degraded (10.934%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:04:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:04:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2038386232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:58 smithi150 ceph-mon[103445]: pgmap v325: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 74 MiB/s wr, 1.19k op/s; 16221/148356 objects degraded (10.934%); 3.6 MiB/s, 57 objects/s recovering 2024-04-05T03:04:58.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:04:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2038386232' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:04:58.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 29 16 54295 54279 116.964 77.6875 0.0105705 0.00854346 2024-04-05T03:04:59.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 30 16 55443 55427 115.457 71.75 0.014839 0.00865406 2024-04-05T03:05:00.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:00 smithi196 ceph-mon[91352]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 16221/158862 objects degraded (10.211%); 3.5 MiB/s, 56 objects/s recovering 2024-04-05T03:05:00.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:00 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1804128707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:00.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:00 smithi150 ceph-mon[103445]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 16221/158862 objects degraded (10.211%); 3.5 MiB/s, 56 objects/s recovering 2024-04-05T03:05:00.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:00 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1804128707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:00.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:00 smithi150 ceph-mon[107701]: pgmap v326: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 16221/158862 objects degraded (10.211%); 3.5 MiB/s, 56 objects/s recovering 2024-04-05T03:05:00.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:00 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1804128707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:00.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 31 16 56652 56636 114.17 75.5625 0.0199374 0.00875258 2024-04-05T03:05:01.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 32 16 57855 57839 112.952 75.1875 0.00678683 0.00884617 2024-04-05T03:05:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:05:01] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.43.0" 2024-04-05T03:05:02.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:02 smithi196 ceph-mon[91352]: pgmap v327: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 16221/158862 objects degraded (10.211%); 3.5 MiB/s, 56 objects/s recovering 2024-04-05T03:05:02.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 16221/158862 objects degraded (10.211%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:02.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:02 smithi150 ceph-mon[103445]: pgmap v327: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 16221/158862 objects degraded (10.211%); 3.5 MiB/s, 56 objects/s recovering 2024-04-05T03:05:02.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 16221/158862 objects degraded (10.211%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:02.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:02 smithi150 ceph-mon[107701]: pgmap v327: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 80 MiB/s wr, 1.29k op/s; 16221/158862 objects degraded (10.211%); 3.5 MiB/s, 56 objects/s recovering 2024-04-05T03:05:02.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 16221/158862 objects degraded (10.211%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:02.885 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 33 16 58953 58937 111.608 68.625 0.0151833 0.00895407 2024-04-05T03:05:03.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:03 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1186429726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:03.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:03 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1186429726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:03.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 34 16 60182 60166 110.585 76.8125 0.0140632 0.00903719 2024-04-05T03:05:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:03 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1186429726' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:04.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:04 smithi150 ceph-mon[103445]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 15915/172158 objects degraded (9.244%); 3.7 MiB/s, 58 objects/s recovering 2024-04-05T03:05:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:04 smithi150 ceph-mon[107701]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 15915/172158 objects degraded (9.244%); 3.7 MiB/s, 58 objects/s recovering 2024-04-05T03:05:04.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 35 16 61318 61302 109.454 71 0.0212517 0.00912832 2024-04-05T03:05:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:04 smithi196 ceph-mon[91352]: pgmap v328: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s; 15915/172158 objects degraded (9.244%); 3.7 MiB/s, 58 objects/s recovering 2024-04-05T03:05:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/746565460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/746565460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:05.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 36 16 62421 62405 108.328 68.9375 0.0147734 0.00922475 2024-04-05T03:05:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/746565460' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:06.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:06 smithi150 ceph-mon[107701]: pgmap v329: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 15915/176715 objects degraded (9.006%); 3.0 MiB/s, 48 objects/s recovering 2024-04-05T03:05:06.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:06 smithi150 ceph-mon[103445]: pgmap v329: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 15915/176715 objects degraded (9.006%); 3.0 MiB/s, 48 objects/s recovering 2024-04-05T03:05:06.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 37 16 63588 63572 107.371 72.9375 0.0129937 0.00930676 2024-04-05T03:05:06.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:06 smithi196 ceph-mon[91352]: pgmap v329: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s; 15915/176715 objects degraded (9.006%); 3.0 MiB/s, 48 objects/s recovering 2024-04-05T03:05:07.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 15915/176715 objects degraded (9.006%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:07.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 15915/176715 objects degraded (9.006%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:07.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 38 16 64619 64603 106.241 64.4375 0.00696099 0.00940613 2024-04-05T03:05:07.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 15915/176715 objects degraded (9.006%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:08 smithi150 ceph-mon[107701]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 15599/184314 objects degraded (8.463%); 4.7 MiB/s, 75 objects/s recovering 2024-04-05T03:05:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:08 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2491376855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:08.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:08 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:05:08.540+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:08.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:08 smithi150 ceph-mon[103445]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 15599/184314 objects degraded (8.463%); 4.7 MiB/s, 75 objects/s recovering 2024-04-05T03:05:08.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:08 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2491376855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:08.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 39 16 65724 65708 105.288 69.0625 0.0193557 0.0094895 2024-04-05T03:05:08.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:08 smithi196 ceph-mon[91352]: pgmap v330: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s; 15599/184314 objects degraded (8.463%); 4.7 MiB/s, 75 objects/s recovering 2024-04-05T03:05:08.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:08 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2491376855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:09 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:09 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:09.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:09 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:09.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:09 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:09.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:09 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:09.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:09 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:09.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:05:09.887777+0000 min lat: 0.000950756 max lat: 2.23538 avg lat: 0.00956727 2024-04-05T03:05:09.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:05:09.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 40 16 66870 66854 104.446 71.625 0.00904455 0.00956727 2024-04-05T03:05:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:09 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:09 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:09.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:09 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:10 smithi150 ceph-mon[107701]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 15599/189648 objects degraded (8.225%); 3.2 MiB/s, 51 objects/s recovering 2024-04-05T03:05:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3106728067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:10 smithi150 ceph-mon[103445]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 15599/189648 objects degraded (8.225%); 3.2 MiB/s, 51 objects/s recovering 2024-04-05T03:05:10.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3106728067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:10.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:10 smithi196 ceph-mon[91352]: pgmap v331: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 15599/189648 objects degraded (8.225%); 3.2 MiB/s, 51 objects/s recovering 2024-04-05T03:05:10.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3106728067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:10.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 41 16 67859 67843 103.406 61.8125 0.0112946 0.00966317 2024-04-05T03:05:11.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:11.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:05:11] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2024-04-05T03:05:11.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:11.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 42 16 68953 68937 102.572 68.375 0.0105519 0.00974228 2024-04-05T03:05:12.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:12 smithi150 ceph-mon[107701]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 15599/193782 objects degraded (8.050%); 3.2 MiB/s, 51 objects/s recovering 2024-04-05T03:05:12.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 15599/193782 objects degraded (8.050%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:12 smithi150 ceph-mon[103445]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 15599/193782 objects degraded (8.050%); 3.2 MiB/s, 51 objects/s recovering 2024-04-05T03:05:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 15599/193782 objects degraded (8.050%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:12.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:12 smithi196 ceph-mon[91352]: pgmap v332: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 969 op/s; 15599/193782 objects degraded (8.050%); 3.2 MiB/s, 51 objects/s recovering 2024-04-05T03:05:12.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 15599/193782 objects degraded (8.050%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:12.886 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 43 16 69942 69926 101.624 61.8125 0.0297086 0.00983338 2024-04-05T03:05:13.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:13 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2732733595' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:13.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:13 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2732733595' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:13.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:13 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2732733595' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:13.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 44 16 70824 70808 100.567 55.125 0.0149511 0.00993697 2024-04-05T03:05:14.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:14 smithi150 ceph-mon[103445]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 15273/205746 objects degraded (7.423%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:14 smithi150 ceph-mon[107701]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 15273/205746 objects degraded (7.423%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:14 smithi196 ceph-mon[91352]: pgmap v333: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s; 15273/205746 objects degraded (7.423%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:14.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 45 16 71816 71800 99.7099 62 0.00851598 0.0100221 2024-04-05T03:05:15.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 46 15 72791 72776 98.8683 61 0.0113888 0.0101081 2024-04-05T03:05:16.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:16 smithi196 ceph-mon[91352]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 15273/209427 objects degraded (7.293%); 3.3 MiB/s, 53 objects/s recovering 2024-04-05T03:05:16.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:16 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1560749157' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:16.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 47 16 73882 73866 98.214 68.125 0.00960663 0.0101752 2024-04-05T03:05:16.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:16 smithi150 ceph-mon[107701]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 15273/209427 objects degraded (7.293%); 3.3 MiB/s, 53 objects/s recovering 2024-04-05T03:05:16.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:16 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1560749157' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:16 smithi150 ceph-mon[103445]: pgmap v334: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 15273/209427 objects degraded (7.293%); 3.3 MiB/s, 53 objects/s recovering 2024-04-05T03:05:16.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:16 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1560749157' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:17.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 15273/209427 objects degraded (7.293%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:17.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 48 16 74891 74875 97.4816 63.0625 0.0126609 0.0102521 2024-04-05T03:05:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 15273/209427 objects degraded (7.293%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:17.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 15273/209427 objects degraded (7.293%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:18.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:18 smithi196 ceph-mon[91352]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 14916/215904 objects degraded (6.909%); 5.2 MiB/s, 83 objects/s recovering 2024-04-05T03:05:18.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:18 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2411883080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:18.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 49 16 75914 75898 96.7969 63.9375 0.00651684 0.0103245 2024-04-05T03:05:18.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:18 smithi150 ceph-mon[107701]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 14916/215904 objects degraded (6.909%); 5.2 MiB/s, 83 objects/s recovering 2024-04-05T03:05:18.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:18 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2411883080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:18.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:18 smithi150 ceph-mon[103445]: pgmap v335: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 14916/215904 objects degraded (6.909%); 5.2 MiB/s, 83 objects/s recovering 2024-04-05T03:05:18.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:18 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2411883080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:19.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 50 16 76979 76963 96.1921 66.5625 0.0156426 0.0103897 2024-04-05T03:05:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:20 smithi196 ceph-mon[91352]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 14916/220590 objects degraded (6.762%); 3.6 MiB/s, 56 objects/s recovering 2024-04-05T03:05:20.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 51 16 77962 77946 95.5106 61.4375 0.0118277 0.010463 2024-04-05T03:05:20.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:20 smithi150 ceph-mon[107701]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 14916/220590 objects degraded (6.762%); 3.6 MiB/s, 56 objects/s recovering 2024-04-05T03:05:20.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:20 smithi150 ceph-mon[103445]: pgmap v336: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 14916/220590 objects degraded (6.762%); 3.6 MiB/s, 56 objects/s recovering 2024-04-05T03:05:21.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:21 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3139724213' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:21.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 52 16 79006 78990 94.9286 65.25 0.0101474 0.0105282 2024-04-05T03:05:21.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:21 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3139724213' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:21.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:05:21] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2024-04-05T03:05:21.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:21 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3139724213' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:22.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:22 smithi196 ceph-mon[91352]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 14916/224460 objects degraded (6.645%); 3.6 MiB/s, 56 objects/s recovering 2024-04-05T03:05:22.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 14916/224460 objects degraded (6.645%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:22.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 53 16 79968 79952 94.2718 60.125 0.00946956 0.0105711 2024-04-05T03:05:22.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:22 smithi150 ceph-mon[107701]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 14916/224460 objects degraded (6.645%); 3.6 MiB/s, 56 objects/s recovering 2024-04-05T03:05:22.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 14916/224460 objects degraded (6.645%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:22 smithi150 ceph-mon[103445]: pgmap v337: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 14916/224460 objects degraded (6.645%); 3.6 MiB/s, 56 objects/s recovering 2024-04-05T03:05:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 14916/224460 objects degraded (6.645%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:23.786 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[107701]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.786 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[107701]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.786 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[107701]: osdmap e122: 8 total, 8 up, 8 in 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[107701]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[107701]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1576961212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:05:23.558+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[103445]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[103445]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[103445]: osdmap e122: 8 total, 8 up, 8 in 2024-04-05T03:05:23.787 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[103445]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.788 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[103445]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.788 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:23 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1576961212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:23 smithi196 ceph-mon[91352]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:23 smithi196 ceph-mon[91352]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:23 smithi196 ceph-mon[91352]: osdmap e122: 8 total, 8 up, 8 in 2024-04-05T03:05:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:23 smithi196 ceph-mon[91352]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:23 smithi196 ceph-mon[91352]: failed to encode map e122 with expected crc 2024-04-05T03:05:23.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:23 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1576961212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:23.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 54 16 80273 80257 92.8791 19.0625 0.0458609 0.0107597 2024-04-05T03:05:24.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 14499/235902 objects degraded (6.146%); 4.8 MiB/s, 77 objects/s recovering 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: osdmap e123: 8 total, 8 up, 8 in 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:24.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:24 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:24.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 55 16 81241 81225 92.2903 60.5 0.0296819 0.0108281 2024-04-05T03:05:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 14499/235902 objects degraded (6.146%); 4.8 MiB/s, 77 objects/s recovering 2024-04-05T03:05:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: osdmap e123: 8 total, 8 up, 8 in 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: pgmap v339: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 14499/235902 objects degraded (6.146%); 4.8 MiB/s, 77 objects/s recovering 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: osdmap e123: 8 total, 8 up, 8 in 2024-04-05T03:05:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: failed to encode map e123 with expected crc 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:24.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:24 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:25.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 56 16 82140 82124 91.6455 56.1875 0.0105907 0.0109057 2024-04-05T03:05:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:26 smithi196 ceph-mon[91352]: pgmap v341: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 14344/239082 objects degraded (6.000%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:05:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:26.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:26 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2509115660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:26.887 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 57 16 83109 83093 91.1001 60.5625 0.0374724 0.0109709 2024-04-05T03:05:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:26 smithi150 ceph-mon[103445]: pgmap v341: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 14344/239082 objects degraded (6.000%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:05:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:26 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2509115660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:26.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:26 smithi150 ceph-mon[107701]: pgmap v341: 89 pgs: 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 60 MiB/s wr, 965 op/s; 14344/239082 objects degraded (6.000%); 4.5 MiB/s, 71 objects/s recovering 2024-04-05T03:05:26.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:26.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:26 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2509115660' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 14344/239082 objects degraded (6.000%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:27.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 58 16 84046 84030 90.5391 58.5625 0.0315869 0.0110382 2024-04-05T03:05:27.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 14344/239082 objects degraded (6.000%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 14344/239082 objects degraded (6.000%), 7 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:28.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:28 smithi196 ceph-mon[91352]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 14290/244458 objects degraded (5.846%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:28.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:28 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/546443699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:28.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 59 16 84925 84909 89.9356 54.9375 0.0107774 0.0111126 2024-04-05T03:05:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:28 smithi150 ceph-mon[103445]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 14290/244458 objects degraded (5.846%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:28 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/546443699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:28.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:28 smithi150 ceph-mon[107701]: pgmap v342: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 14290/244458 objects degraded (5.846%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:28.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:28 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/546443699' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:29.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:05:29.889573+0000 min lat: 0.000950756 max lat: 2.23538 avg lat: 0.0111756 2024-04-05T03:05:29.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:05:29.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 60 16 85879 85863 89.4303 59.625 0.0104478 0.0111756 2024-04-05T03:05:30.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:30 smithi196 ceph-mon[91352]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 14290/248646 objects degraded (5.747%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:30.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 61 16 86788 86772 88.8955 56.8125 0.00595127 0.0112433 2024-04-05T03:05:30.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:30 smithi150 ceph-mon[103445]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 14290/248646 objects degraded (5.747%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:30.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:30 smithi150 ceph-mon[107701]: pgmap v343: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 14290/248646 objects degraded (5.747%); 4.9 MiB/s, 78 objects/s recovering 2024-04-05T03:05:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:31 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2956603892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:31.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 62 16 87715 87699 88.3961 57.9375 0.0313593 0.0113063 2024-04-05T03:05:31.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:05:31] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2024-04-05T03:05:31.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:31 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2956603892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:31.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:31 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2956603892' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:32.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:32 smithi196 ceph-mon[91352]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 14115/252117 objects degraded (5.599%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:05:32.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 14115/252117 objects degraded (5.599%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:32.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 63 16 88672 88656 87.9424 59.8125 0.00710373 0.011365 2024-04-05T03:05:32.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:32 smithi150 ceph-mon[103445]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 14115/252117 objects degraded (5.599%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:05:32.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 14115/252117 objects degraded (5.599%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:32 smithi150 ceph-mon[107701]: pgmap v344: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 749 op/s; 14115/252117 objects degraded (5.599%); 2.7 MiB/s, 43 objects/s recovering 2024-04-05T03:05:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 14115/252117 objects degraded (5.599%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:33.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:33 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/112862494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:33.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 64 15 89526 89511 87.4032 53.4375 0.0280279 0.0114357 2024-04-05T03:05:33.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:33 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/112862494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:33.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:33 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/112862494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:34.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:34 smithi196 ceph-mon[91352]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s; 14115/262461 objects degraded (5.378%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:05:34.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 65 16 90432 90416 86.9286 56.5625 0.0220664 0.0114968 2024-04-05T03:05:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:34 smithi150 ceph-mon[107701]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s; 14115/262461 objects degraded (5.378%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:05:34.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:34 smithi150 ceph-mon[103445]: pgmap v345: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s; 14115/262461 objects degraded (5.378%); 2.4 MiB/s, 38 objects/s recovering 2024-04-05T03:05:35.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 66 16 91365 91349 86.495 58.3125 0.00902505 0.0115537 2024-04-05T03:05:36.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:36 smithi196 ceph-mon[91352]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 13958/265917 objects degraded (5.249%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:05:36.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:36 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1631937796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:36.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 67 16 92272 92256 86.05 56.6875 0.0185078 0.0116149 2024-04-05T03:05:36.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:36 smithi150 ceph-mon[107701]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 13958/265917 objects degraded (5.249%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:05:36.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:36 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1631937796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:36.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:36 smithi150 ceph-mon[103445]: pgmap v346: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 13958/265917 objects degraded (5.249%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:05:36.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:36 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1631937796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 13958/265917 objects degraded (5.249%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:37.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 68 16 93221 93205 85.6567 59.3125 0.0120347 0.0116674 2024-04-05T03:05:37.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 13958/265917 objects degraded (5.249%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:37.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 13958/265917 objects degraded (5.249%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:38 smithi196 ceph-mon[91352]: pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 13958/271953 objects degraded (5.133%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:05:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:38 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:38.888 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 69 16 94086 94070 85.1988 54.0625 0.0241601 0.011731 2024-04-05T03:05:38.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:38 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:05:38.579+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:38 smithi150 ceph-mon[107701]: pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 13958/271953 objects degraded (5.133%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:05:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:38 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:38.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:38 smithi150 ceph-mon[103445]: pgmap v347: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s; 13958/271953 objects degraded (5.133%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:05:38.979 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:38 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:39.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 70 16 94983 94967 84.7825 56.0625 0.00489791 0.0117891 2024-04-05T03:05:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:39 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:39 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:39 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1398403123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:39 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:39 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:39.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:39 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1398403123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:40.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:39 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:05:40.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:39 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:05:40.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:39 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1398403123' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:40.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 71 16 95841 95825 84.3436 53.625 0.0502672 0.0118491 2024-04-05T03:05:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:40 smithi150 ceph-mon[107701]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 13958/276141 objects degraded (5.055%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:05:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:40 smithi150 ceph-mon[103445]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 13958/276141 objects degraded (5.055%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:05:41.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:40 smithi196 ceph-mon[91352]: pgmap v348: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 13958/276141 objects degraded (5.055%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:05:41.879 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:41.879 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-mon[103445]: pgmap v349: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 13784/279606 objects degraded (4.930%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:05:41.880 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1582184329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:41.880 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:05:41] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2024-04-05T03:05:41.880 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:41.880 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-mon[107701]: pgmap v349: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 13784/279606 objects degraded (4.930%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:05:41.880 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1582184329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:41.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 72 16 96695 96679 83.9134 53.375 0.00492051 0.0119112 2024-04-05T03:05:42.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:42.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:41 smithi196 ceph-mon[91352]: pgmap v349: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 13784/279606 objects degraded (4.930%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:05:42.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1582184329' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:42.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 73 16 97605 97589 83.543 56.875 0.00483527 0.0119627 2024-04-05T03:05:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:42 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 13784/279606 objects degraded (4.930%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:42.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:42 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 13784/279606 objects degraded (4.930%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:42 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 13784/279606 objects degraded (4.930%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:43.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 74 16 98399 98383 83.0846 49.625 0.0228268 0.0120286 2024-04-05T03:05:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:43 smithi150 ceph-mon[107701]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 13784/289443 objects degraded (4.762%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:05:43.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:43 smithi150 ceph-mon[103445]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 13784/289443 objects degraded (4.762%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:05:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:43 smithi196 ceph-mon[91352]: pgmap v350: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 13784/289443 objects degraded (4.762%); 1.7 MiB/s, 27 objects/s recovering 2024-04-05T03:05:44.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 75 16 99271 99255 82.7034 54.5 0.0319173 0.0120851 2024-04-05T03:05:44.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:44 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/776810395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:44.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:44 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/776810395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:45.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:44 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/776810395' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:45.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 76 16 100164 100148 82.3495 55.8125 0.0546688 0.0121372 2024-04-05T03:05:45.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:45 smithi150 ceph-mon[103445]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 13607/292695 objects degraded (4.649%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:05:45.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:45 smithi150 ceph-mon[107701]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 13607/292695 objects degraded (4.649%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:05:46.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:45 smithi196 ceph-mon[91352]: pgmap v351: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 839 op/s; 13607/292695 objects degraded (4.649%); 2.6 MiB/s, 42 objects/s recovering 2024-04-05T03:05:46.889 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 77 16 100976 100960 81.9391 50.75 0.0213716 0.012198 2024-04-05T03:05:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:46 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2285083839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:46.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:46 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2285083839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:47.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:46 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2285083839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:47.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 78 16 101853 101837 81.5912 54.8125 0.00636295 0.0122501 2024-04-05T03:05:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 13607/292695 objects degraded (4.649%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:47 smithi150 ceph-mon[103445]: pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 13607/298254 objects degraded (4.562%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:05:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 13607/292695 objects degraded (4.649%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:47 smithi150 ceph-mon[107701]: pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 13607/298254 objects degraded (4.562%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:05:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 13607/292695 objects degraded (4.649%), 6 pgs degraded (PG_DEGRADED) 2024-04-05T03:05:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:47 smithi196 ceph-mon[91352]: pgmap v352: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 13607/298254 objects degraded (4.562%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:05:48.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 79 16 102699 102683 81.2277 52.875 0.0150157 0.0123042 2024-04-05T03:05:49.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:49 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2627857290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:49.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:49 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2627857290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:49.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:49 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2627857290' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:49.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:2024-04-05T03:05:49.891383+0000 min lat: 0.000950756 max lat: 2.23538 avg lat: 0.0123602 2024-04-05T03:05:49.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-04-05T03:05:49.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 80 16 103524 103508 80.8568 51.5625 0.0417957 0.0123602 2024-04-05T03:05:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:50 smithi150 ceph-mon[107701]: pgmap v353: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 13607/302322 objects degraded (4.501%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:05:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:50 smithi150 ceph-mon[103445]: pgmap v353: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 13607/302322 objects degraded (4.501%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:05:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:50 smithi196 ceph-mon[91352]: pgmap v353: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 13607/302322 objects degraded (4.501%); 1.8 MiB/s, 29 objects/s recovering 2024-04-05T03:05:50.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 81 16 104377 104361 80.5167 53.3125 0.0269519 0.0124141 2024-04-05T03:05:51.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 82 16 105158 105142 80.13 48.8125 0.0123355 0.0124731 2024-04-05T03:05:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:05:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:05:51] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2024-04-05T03:05:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:52 smithi150 ceph-mon[103445]: pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 13427/305616 objects degraded (4.393%); 2.8 MiB/s, 44 objects/s recovering 2024-04-05T03:05:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1889815147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:52.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 13427/305616 objects degraded (4.393%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:05:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:52 smithi150 ceph-mon[107701]: pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 13427/305616 objects degraded (4.393%); 2.8 MiB/s, 44 objects/s recovering 2024-04-05T03:05:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1889815147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 13427/305616 objects degraded (4.393%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:05:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:52 smithi196 ceph-mon[91352]: pgmap v354: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 818 op/s; 13427/305616 objects degraded (4.393%); 2.8 MiB/s, 44 objects/s recovering 2024-04-05T03:05:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1889815147' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 13427/305616 objects degraded (4.393%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:05:52.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 83 16 106003 105987 79.8008 52.8125 0.0149464 0.0125244 2024-04-05T03:05:53.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 84 16 106802 106786 79.4453 49.9375 0.0690954 0.0125785 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[103445]: pgmap v355: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 13427/314898 objects degraded (4.264%); 1.9 MiB/s, 29 objects/s recovering 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3548100598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[107701]: pgmap v355: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 13427/314898 objects degraded (4.264%); 1.9 MiB/s, 29 objects/s recovering 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:05:54.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:05:54.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:54 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3548100598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:54 smithi196 ceph-mon[91352]: pgmap v355: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s; 13427/314898 objects degraded (4.264%); 1.9 MiB/s, 29 objects/s recovering 2024-04-05T03:05:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:54 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:05:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:54 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:05:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:54 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:05:54.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:54 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3548100598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:54.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 85 16 107502 107486 79.0251 43.75 0.0157692 0.0126478 2024-04-05T03:05:55.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 86 16 108306 108290 78.6905 50.25 0.0182696 0.0127019 2024-04-05T03:05:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:56 smithi196 ceph-mon[91352]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 13226/317970 objects degraded (4.160%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:05:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:56 smithi150 ceph-mon[107701]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 13226/317970 objects degraded (4.160%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:05:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:56 smithi150 ceph-mon[103445]: pgmap v356: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 50 MiB/s wr, 792 op/s; 13226/317970 objects degraded (4.160%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:05:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:05:56.890 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 87 16 109003 108987 78.2867 43.5625 0.0307023 0.0127665 2024-04-05T03:05:57.518 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:57 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4011244234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:57.518 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 13226/317970 objects degraded (4.160%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:05:57.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:57 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4011244234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:57.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 13226/317970 objects degraded (4.160%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:05:57.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:57 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4011244234' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:05:57.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 13226/317970 objects degraded (4.160%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:05:57.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 88 16 109794 109778 77.9588 49.4375 0.0200527 0.0128213 2024-04-05T03:05:58.713 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:05:58 smithi196 ceph-mon[91352]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 13226/323106 objects degraded (4.093%); 2.0 MiB/s, 31 objects/s recovering 2024-04-05T03:05:58.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:05:58 smithi150 ceph-mon[103445]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 13226/323106 objects degraded (4.093%); 2.0 MiB/s, 31 objects/s recovering 2024-04-05T03:05:58.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:05:58 smithi150 ceph-mon[107701]: pgmap v357: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s; 13226/323106 objects degraded (4.093%); 2.0 MiB/s, 31 objects/s recovering 2024-04-05T03:05:58.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 89 16 110596 110580 77.646 50.125 0.00586266 0.0128718 2024-04-05T03:05:59.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout: 90 6 111255 111249 77.2478 41.8125 0.0255214 0.0129392 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[103445]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 13226/326517 objects degraded (4.051%); 2.0 MiB/s, 31 objects/s recovering 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2405019820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.356 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.357 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.357 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[107701]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 13226/326517 objects degraded (4.051%); 2.0 MiB/s, 31 objects/s recovering 2024-04-05T03:06:00.357 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2405019820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:00 smithi196 ceph-mon[91352]: pgmap v358: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 13226/326517 objects degraded (4.051%); 2.0 MiB/s, 31 objects/s recovering 2024-04-05T03:06:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:00 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2405019820' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:00.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:06:00.355+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total time run: 90.0212 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Total writes made: 111255 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Write size: 65536 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Object size: 65536 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Bandwidth (MB/sec): 77.2422 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Bandwidth: 48.092 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max bandwidth (MB/sec): 280.375 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min bandwidth (MB/sec): 0 2024-04-05T03:06:00.891 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average IOPS: 1235 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev IOPS: 769.472 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max IOPS: 4486 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min IOPS: 0 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Average Latency(s): 0.01294 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Stddev Latency(s): 0.0299732 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Max latency(s): 2.23538 2024-04-05T03:06:00.892 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Min latency(s): 0.000950756 2024-04-05T03:06:00.898 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Cleaning up (deleting benchmark objects) 2024-04-05T03:06:01.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:01.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 13015/329403 objects degraded (3.951%); 3.4 MiB/s, 54 objects/s recovering 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 13015/329403 objects degraded (3.951%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:01.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:01 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:01.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:01.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 13015/329403 objects degraded (3.951%); 3.4 MiB/s, 54 objects/s recovering 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 13015/329403 objects degraded (3.951%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:01.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: pgmap v359: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 52 MiB/s wr, 832 op/s; 13015/329403 objects degraded (3.951%); 3.4 MiB/s, 54 objects/s recovering 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: pgmap v360: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s; 13015/329403 objects degraded (3.951%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:01.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:06:01] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:06:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 13015/329403 objects degraded (3.951%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:02.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 13015/329403 objects degraded (3.951%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:02.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 13015/329403 objects degraded (3.951%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:03.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:03 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4232276182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:03.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:03 smithi196 ceph-mon[91352]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 40 MiB/s wr, 678 op/s; 13015/331242 objects degraded (3.929%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:06:03.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:03 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4232276182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:03.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:03 smithi150 ceph-mon[107701]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 40 MiB/s wr, 678 op/s; 13015/331242 objects degraded (3.929%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:06:03.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:03 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4232276182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:03.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:03 smithi150 ceph-mon[103445]: pgmap v361: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 40 MiB/s wr, 678 op/s; 13015/331242 objects degraded (3.929%); 2.9 MiB/s, 46 objects/s recovering 2024-04-05T03:06:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:05 smithi196 ceph-mon[91352]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 37 MiB/s wr, 697 op/s; 13015/330861 objects degraded (3.934%); 1.5 MiB/s, 23 objects/s recovering 2024-04-05T03:06:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/777236177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:05 smithi150 ceph-mon[103445]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 37 MiB/s wr, 697 op/s; 13015/330861 objects degraded (3.934%); 1.5 MiB/s, 23 objects/s recovering 2024-04-05T03:06:05.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/777236177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:05 smithi150 ceph-mon[107701]: pgmap v362: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 37 MiB/s wr, 697 op/s; 13015/330861 objects degraded (3.934%); 1.5 MiB/s, 23 objects/s recovering 2024-04-05T03:06:05.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/777236177' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:07 smithi196 ceph-mon[91352]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 25 MiB/s wr, 598 op/s; 12574/328713 objects degraded (3.825%); 4.6 MiB/s, 73 objects/s recovering 2024-04-05T03:06:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 12574/328713 objects degraded (3.825%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:07.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:07 smithi150 ceph-mon[103445]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 25 MiB/s wr, 598 op/s; 12574/328713 objects degraded (3.825%); 4.6 MiB/s, 73 objects/s recovering 2024-04-05T03:06:07.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 12574/328713 objects degraded (3.825%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:07.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:07 smithi150 ceph-mon[107701]: pgmap v363: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 25 MiB/s wr, 598 op/s; 12574/328713 objects degraded (3.825%); 4.6 MiB/s, 73 objects/s recovering 2024-04-05T03:06:07.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 12574/328713 objects degraded (3.825%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:08.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:08 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4286914887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:08.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:08 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4286914887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:08.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:08 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4286914887' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:09.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:09 smithi196 ceph-mon[91352]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 17 MiB/s wr, 708 op/s; 12574/322374 objects degraded (3.900%); 4.6 MiB/s, 73 objects/s recovering 2024-04-05T03:06:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:09 smithi150 ceph-mon[103445]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 17 MiB/s wr, 708 op/s; 12574/322374 objects degraded (3.900%); 4.6 MiB/s, 73 objects/s recovering 2024-04-05T03:06:09.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:09 smithi150 ceph-mon[107701]: pgmap v364: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 115 B/s rd, 17 MiB/s wr, 708 op/s; 12574/322374 objects degraded (3.900%); 4.6 MiB/s, 73 objects/s recovering 2024-04-05T03:06:10.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2239781360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:10.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2239781360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2239781360' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:11.628 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:11 smithi150 ceph-mon[107701]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 9.2 MiB/s wr, 739 op/s; 12313/316101 objects degraded (3.895%); 4.4 MiB/s, 70 objects/s recovering 2024-04-05T03:06:11.628 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:11.629 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:11 smithi150 ceph-mon[103445]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 9.2 MiB/s wr, 739 op/s; 12313/316101 objects degraded (3.895%); 4.4 MiB/s, 70 objects/s recovering 2024-04-05T03:06:11.629 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:11.629 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:06:11] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:06:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:11 smithi196 ceph-mon[91352]: pgmap v365: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 9.2 MiB/s wr, 739 op/s; 12313/316101 objects degraded (3.895%); 4.4 MiB/s, 70 objects/s recovering 2024-04-05T03:06:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:12.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 12313/316101 objects degraded (3.895%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:12.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 12313/316101 objects degraded (3.895%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:12.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 12313/316101 objects degraded (3.895%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:13.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:13 smithi196 ceph-mon[91352]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 746 op/s; 12313/311397 objects degraded (3.954%); 3.7 MiB/s, 58 objects/s recovering 2024-04-05T03:06:13.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:13 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3698632194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:13.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:13 smithi150 ceph-mon[107701]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 746 op/s; 12313/311397 objects degraded (3.954%); 3.7 MiB/s, 58 objects/s recovering 2024-04-05T03:06:13.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:13 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3698632194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:13.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:13 smithi150 ceph-mon[103445]: pgmap v366: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 85 B/s rd, 7.7 MiB/s wr, 746 op/s; 12313/311397 objects degraded (3.954%); 3.7 MiB/s, 58 objects/s recovering 2024-04-05T03:06:13.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:13 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3698632194' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: osdmap e124: 8 total, 8 up, 8 in 2024-04-05T03:06:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:14 smithi196 ceph-mon[91352]: osdmap e125: 8 total, 8 up, 8 in 2024-04-05T03:06:14.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: osdmap e124: 8 total, 8 up, 8 in 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[107701]: osdmap e125: 8 total, 8 up, 8 in 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: osdmap e124: 8 total, 8 up, 8 in 2024-04-05T03:06:14.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: failed to encode map e124 with expected crc 2024-04-05T03:06:14.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:14 smithi150 ceph-mon[103445]: osdmap e125: 8 total, 8 up, 8 in 2024-04-05T03:06:15.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:15 smithi196 ceph-mon[91352]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 377 KiB/s wr, 815 op/s; 12313/306741 objects degraded (4.014%); 4.4 MiB/s, 70 objects/s recovering 2024-04-05T03:06:15.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:15 smithi196 ceph-mon[91352]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:15 smithi196 ceph-mon[91352]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:15 smithi196 ceph-mon[91352]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:15 smithi196 ceph-mon[91352]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:15 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1897768545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:15.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:06:15.371+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[107701]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 377 KiB/s wr, 815 op/s; 12313/306741 objects degraded (4.014%); 4.4 MiB/s, 70 objects/s recovering 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[107701]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[107701]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[107701]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[107701]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1897768545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[103445]: pgmap v368: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 377 KiB/s wr, 815 op/s; 12313/306741 objects degraded (4.014%); 4.4 MiB/s, 70 objects/s recovering 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[103445]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[103445]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[103445]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[103445]: failed to encode map e125 with expected crc 2024-04-05T03:06:15.729 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:15 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1897768545' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:16.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:16 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:16.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:16 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:16.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:16 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:16.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:16 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:16.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:16 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:16.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:16 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:16 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:16 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:16.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:16 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-04-05T03:06:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:17 smithi196 ceph-mon[91352]: pgmap v370: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 12313/304776 objects degraded (4.040%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 12313/304776 objects degraded (4.040%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:17.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:17 smithi150 ceph-mon[107701]: pgmap v370: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 12313/304776 objects degraded (4.040%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:17.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 12313/304776 objects degraded (4.040%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:17 smithi150 ceph-mon[103445]: pgmap v370: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 12313/304776 objects degraded (4.040%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 12313/304776 objects degraded (4.040%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:18 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2821038635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:18 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2821038635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:18.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:18 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2821038635' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:19 smithi196 ceph-mon[91352]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 11867/298608 objects degraded (3.974%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:06:19.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:19 smithi150 ceph-mon[107701]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 11867/298608 objects degraded (3.974%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:06:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:19 smithi150 ceph-mon[103445]: pgmap v371: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 11867/298608 objects degraded (3.974%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:06:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:20 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/673402986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:20.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:20 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/673402986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:20.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:20 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/673402986' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:21 smithi196 ceph-mon[91352]: pgmap v372: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 11867/293922 objects degraded (4.037%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:06:21.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:21 smithi150 ceph-mon[107701]: pgmap v372: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 11867/293922 objects degraded (4.037%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:06:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:06:21] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:06:21.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:21 smithi150 ceph-mon[103445]: pgmap v372: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 11867/293922 objects degraded (4.037%); 3.5 MiB/s, 55 objects/s recovering 2024-04-05T03:06:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 11867/293922 objects degraded (4.037%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:22.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 11867/293922 objects degraded (4.037%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:22.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 11867/293922 objects degraded (4.037%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:23.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:23 smithi196 ceph-mon[91352]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 11867/289539 objects degraded (4.099%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:06:23.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:23 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/94909695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:23.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:23 smithi150 ceph-mon[103445]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 11867/289539 objects degraded (4.099%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:06:23.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:23 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/94909695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:23.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:23 smithi150 ceph-mon[107701]: pgmap v373: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 11867/289539 objects degraded (4.099%); 3.1 MiB/s, 50 objects/s recovering 2024-04-05T03:06:23.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:23 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/94909695' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:25 smithi150 ceph-mon[103445]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 11690/284193 objects degraded (4.113%); 3.9 MiB/s, 62 objects/s recovering 2024-04-05T03:06:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:25 smithi150 ceph-mon[107701]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 11690/284193 objects degraded (4.113%); 3.9 MiB/s, 62 objects/s recovering 2024-04-05T03:06:25.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:25 smithi196 ceph-mon[91352]: pgmap v374: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 11690/284193 objects degraded (4.113%); 3.9 MiB/s, 62 objects/s recovering 2024-04-05T03:06:26.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:26 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1123304796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:26.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:26 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1123304796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:26 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1123304796' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:27 smithi150 ceph-mon[103445]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 11690/281949 objects degraded (4.146%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:06:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 11690/281949 objects degraded (4.146%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:27.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:27 smithi150 ceph-mon[107701]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 11690/281949 objects degraded (4.146%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:06:27.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 11690/281949 objects degraded (4.146%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:27 smithi196 ceph-mon[91352]: pgmap v375: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 11690/281949 objects degraded (4.146%); 3.3 MiB/s, 52 objects/s recovering 2024-04-05T03:06:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 11690/281949 objects degraded (4.146%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:28.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:28 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/648266518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:28.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:28 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/648266518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:28.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:28 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/648266518' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:29 smithi150 ceph-mon[107701]: pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 11506/275412 objects degraded (4.178%); 4.2 MiB/s, 67 objects/s recovering 2024-04-05T03:06:29.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:29 smithi150 ceph-mon[103445]: pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 11506/275412 objects degraded (4.178%); 4.2 MiB/s, 67 objects/s recovering 2024-04-05T03:06:29.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:29 smithi196 ceph-mon[91352]: pgmap v376: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 11506/275412 objects degraded (4.178%); 4.2 MiB/s, 67 objects/s recovering 2024-04-05T03:06:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:30 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:30.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:06:30.387+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:30 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:30.787 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:30 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[103445]: pgmap v377: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 11506/270333 objects degraded (4.256%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2432232026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:31.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:06:31] "GET /metrics HTTP/1.1" 200 36498 "" "Prometheus/2.43.0" 2024-04-05T03:06:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[107701]: pgmap v377: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 11506/270333 objects degraded (4.256%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:31 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2432232026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:31 smithi196 ceph-mon[91352]: pgmap v377: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 11506/270333 objects degraded (4.256%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:31 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:31 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:31 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2432232026' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 11506/270333 objects degraded (4.256%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:32.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 11506/270333 objects degraded (4.256%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:32.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 11506/270333 objects degraded (4.256%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:33 smithi150 ceph-mon[103445]: pgmap v378: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11506/265914 objects degraded (4.327%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:33 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1453390285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:33 smithi150 ceph-mon[107701]: pgmap v378: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11506/265914 objects degraded (4.327%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:33 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1453390285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:33.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:33 smithi196 ceph-mon[91352]: pgmap v378: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 777 op/s; 11506/265914 objects degraded (4.327%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:33.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:33 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1453390285' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:35.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:35 smithi150 ceph-mon[103445]: pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 11318/260991 objects degraded (4.337%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:06:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:35 smithi150 ceph-mon[107701]: pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 11318/260991 objects degraded (4.337%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:06:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:35 smithi196 ceph-mon[91352]: pgmap v379: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 11318/260991 objects degraded (4.337%); 2.9 MiB/s, 45 objects/s recovering 2024-04-05T03:06:36.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:36 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1855258170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:36.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:36 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1855258170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:36.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:36 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1855258170' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:37 smithi150 ceph-mon[103445]: pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 11318/258810 objects degraded (4.373%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 11318/258810 objects degraded (4.373%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:37.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:37 smithi150 ceph-mon[107701]: pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 11318/258810 objects degraded (4.373%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:37.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 11318/258810 objects degraded (4.373%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:37 smithi196 ceph-mon[91352]: pgmap v380: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 11318/258810 objects degraded (4.373%); 1.9 MiB/s, 30 objects/s recovering 2024-04-05T03:06:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 11318/258810 objects degraded (4.373%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:39.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:39 smithi150 ceph-mon[103445]: pgmap v381: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 11119/252687 objects degraded (4.400%); 3.0 MiB/s, 47 objects/s recovering 2024-04-05T03:06:39.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:39 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2331620166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:39 smithi150 ceph-mon[107701]: pgmap v381: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 11119/252687 objects degraded (4.400%); 3.0 MiB/s, 47 objects/s recovering 2024-04-05T03:06:39.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:39 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2331620166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:39 smithi196 ceph-mon[91352]: pgmap v381: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 11119/252687 objects degraded (4.400%); 3.0 MiB/s, 47 objects/s recovering 2024-04-05T03:06:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:39 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2331620166' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-mon[103445]: pgmap v382: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 11119/247476 objects degraded (4.493%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2125367874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:41.728 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:06:41] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:06:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-mon[107701]: pgmap v382: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 11119/247476 objects degraded (4.493%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2125367874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:41 smithi196 ceph-mon[91352]: pgmap v382: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 11119/247476 objects degraded (4.493%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2125367874' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:42.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:42 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 11119/247476 objects degraded (4.493%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:42.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:42 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 11119/247476 objects degraded (4.493%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:42 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 11119/247476 objects degraded (4.493%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:43.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:43 smithi196 ceph-mon[91352]: pgmap v383: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11119/243312 objects degraded (4.570%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:43.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:43 smithi150 ceph-mon[103445]: pgmap v383: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11119/243312 objects degraded (4.570%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:43 smithi150 ceph-mon[107701]: pgmap v383: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 11119/243312 objects degraded (4.570%); 2.0 MiB/s, 32 objects/s recovering 2024-04-05T03:06:44.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:44 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2284220489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:44.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:44 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2284220489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:44.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:44 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2284220489' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:45.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:45 smithi150 ceph-mon[107701]: pgmap v384: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 10911/238218 objects degraded (4.580%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:06:45.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:45 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:45.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:45 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:06:45.413+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:45.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:45 smithi150 ceph-mon[103445]: pgmap v384: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 10911/238218 objects degraded (4.580%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:06:45.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:45 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:45 smithi196 ceph-mon[91352]: pgmap v384: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 10911/238218 objects degraded (4.580%); 3.1 MiB/s, 49 objects/s recovering 2024-04-05T03:06:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:45 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:46.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:46 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:46.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:46 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:46.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:46 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4261493202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:46 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:46 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:46 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4261493202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:46 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:06:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:46 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:06:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:46 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4261493202' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:47.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:47 smithi196 ceph-mon[91352]: pgmap v385: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 10911/236199 objects degraded (4.619%); 2.1 MiB/s, 33 objects/s recovering 2024-04-05T03:06:47.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 10911/236199 objects degraded (4.619%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:47 smithi150 ceph-mon[103445]: pgmap v385: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 10911/236199 objects degraded (4.619%); 2.1 MiB/s, 33 objects/s recovering 2024-04-05T03:06:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 10911/236199 objects degraded (4.619%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:47 smithi150 ceph-mon[107701]: pgmap v385: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 10911/236199 objects degraded (4.619%); 2.1 MiB/s, 33 objects/s recovering 2024-04-05T03:06:47.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 10911/236199 objects degraded (4.619%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:49.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:49 smithi196 ceph-mon[91352]: pgmap v386: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 10709/229998 objects degraded (4.656%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:06:49.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:49 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/523003783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:49.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:49 smithi150 ceph-mon[103445]: pgmap v386: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 10709/229998 objects degraded (4.656%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:06:49.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:49 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/523003783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:49.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:49 smithi150 ceph-mon[107701]: pgmap v386: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 10709/229998 objects degraded (4.656%); 3.2 MiB/s, 50 objects/s recovering 2024-04-05T03:06:49.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:49 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/523003783' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:51.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:51 smithi196 ceph-mon[91352]: pgmap v387: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 10709/225093 objects degraded (4.758%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:06:51.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:51 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3423862118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:51.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:51 smithi150 ceph-mon[103445]: pgmap v387: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 10709/225093 objects degraded (4.758%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:06:51.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:51 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3423862118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:06:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:06:51] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:06:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:51 smithi150 ceph-mon[107701]: pgmap v387: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 10709/225093 objects degraded (4.758%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:06:51.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:51 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3423862118' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 10709/225093 objects degraded (4.758%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:52.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 10709/225093 objects degraded (4.758%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:52.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 10709/225093 objects degraded (4.758%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:53.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:53 smithi196 ceph-mon[91352]: pgmap v388: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 10709/220947 objects degraded (4.847%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:06:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:53 smithi150 ceph-mon[103445]: pgmap v388: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 10709/220947 objects degraded (4.847%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:06:53.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:53 smithi150 ceph-mon[107701]: pgmap v388: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 10709/220947 objects degraded (4.847%); 2.1 MiB/s, 34 objects/s recovering 2024-04-05T03:06:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:54 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3186591498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:54 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3186591498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:54.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:54 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3186591498' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:55.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:55 smithi196 ceph-mon[91352]: pgmap v389: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 10459/216072 objects degraded (4.841%); 2.6 MiB/s, 54 objects/s recovering 2024-04-05T03:06:55.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:55 smithi150 ceph-mon[103445]: pgmap v389: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 10459/216072 objects degraded (4.841%); 2.6 MiB/s, 54 objects/s recovering 2024-04-05T03:06:55.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:55 smithi150 ceph-mon[107701]: pgmap v389: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 10459/216072 objects degraded (4.841%); 2.6 MiB/s, 54 objects/s recovering 2024-04-05T03:06:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:56.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:06:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:57 smithi196 ceph-mon[91352]: pgmap v390: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 10459/213891 objects degraded (4.890%); 1.5 MiB/s, 37 objects/s recovering 2024-04-05T03:06:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:57 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3314273738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 10459/213891 objects degraded (4.890%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:57 smithi150 ceph-mon[103445]: pgmap v390: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 10459/213891 objects degraded (4.890%); 1.5 MiB/s, 37 objects/s recovering 2024-04-05T03:06:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:57 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3314273738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 10459/213891 objects degraded (4.890%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:57.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:57 smithi150 ceph-mon[107701]: pgmap v390: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 10459/213891 objects degraded (4.890%); 1.5 MiB/s, 37 objects/s recovering 2024-04-05T03:06:57.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:57 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3314273738' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:57.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 10459/213891 objects degraded (4.890%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:06:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:59 smithi196 ceph-mon[91352]: pgmap v391: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 10190/207606 objects degraded (4.908%); 1.5 MiB/s, 60 objects/s recovering 2024-04-05T03:06:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:06:59 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1798960493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:59.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:59 smithi150 ceph-mon[103445]: pgmap v391: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 10190/207606 objects degraded (4.908%); 1.5 MiB/s, 60 objects/s recovering 2024-04-05T03:06:59.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:06:59 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1798960493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:06:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:59 smithi150 ceph-mon[107701]: pgmap v391: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 10190/207606 objects degraded (4.908%); 1.5 MiB/s, 60 objects/s recovering 2024-04-05T03:06:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:06:59 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1798960493' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:00.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:00 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:00.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:00 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:00 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:01 smithi196 ceph-mon[91352]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 10190/202671 objects degraded (5.028%); 469 KiB/s, 43 objects/s recovering 2024-04-05T03:07:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:07:01] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2024-04-05T03:07:01.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:01 smithi150 ceph-mon[107701]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 10190/202671 objects degraded (5.028%); 469 KiB/s, 43 objects/s recovering 2024-04-05T03:07:01.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:01 smithi150 ceph-mon[103445]: pgmap v392: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 10190/202671 objects degraded (5.028%); 469 KiB/s, 43 objects/s recovering 2024-04-05T03:07:02.716 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:02 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4172742847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:02.717 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 10190/202671 objects degraded (5.028%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:02.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:02 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4172742847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:02.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 10190/202671 objects degraded (5.028%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:02.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:02 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4172742847' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:02.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 10190/202671 objects degraded (5.028%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:03.860 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:03 smithi150 ceph-mon[107701]: pgmap v393: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10190/198495 objects degraded (5.134%); 469 KiB/s, 43 objects/s recovering 2024-04-05T03:07:03.860 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:03 smithi150 ceph-mon[103445]: pgmap v393: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10190/198495 objects degraded (5.134%); 469 KiB/s, 43 objects/s recovering 2024-04-05T03:07:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:03 smithi196 ceph-mon[91352]: pgmap v393: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 10190/198495 objects degraded (5.134%); 469 KiB/s, 43 objects/s recovering 2024-04-05T03:07:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:04 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/685792437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:04.914 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:04 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/685792437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:04.914 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:04 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/685792437' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:05 smithi196 ceph-mon[91352]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 9951/193695 objects degraded (5.137%); 469 KiB/s, 63 objects/s recovering 2024-04-05T03:07:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:05 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:05 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:05 smithi150 ceph-mon[103445]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 9951/193695 objects degraded (5.137%); 469 KiB/s, 63 objects/s recovering 2024-04-05T03:07:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:05 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:05 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:05.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:05 smithi150 ceph-mon[107701]: pgmap v394: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 9951/193695 objects degraded (5.137%); 469 KiB/s, 63 objects/s recovering 2024-04-05T03:07:05.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:05 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:05.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:05 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: pgmap v395: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9951/191835 objects degraded (5.187%); 0 B/s, 42 objects/s recovering 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: osdmap e126: 8 total, 8 up, 8 in 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: failed to encode map e126 with expected crc 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: failed to encode map e126 with expected crc 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1173552080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9951/191835 objects degraded (5.187%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:07.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:07 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:07:07.730+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:07:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: pgmap v395: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9951/191835 objects degraded (5.187%); 0 B/s, 42 objects/s recovering 2024-04-05T03:07:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: osdmap e126: 8 total, 8 up, 8 in 2024-04-05T03:07:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: failed to encode map e126 with expected crc 2024-04-05T03:07:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: failed to encode map e126 with expected crc 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1173552080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9951/191835 objects degraded (5.187%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: pgmap v395: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 9951/191835 objects degraded (5.187%); 0 B/s, 42 objects/s recovering 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: osdmap e126: 8 total, 8 up, 8 in 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: failed to encode map e126 with expected crc 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: failed to encode map e126 with expected crc 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:07.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1173552080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9951/191835 objects degraded (5.187%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:07:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:07:07.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:07 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9951/187995 objects degraded (5.293%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9951/187995 objects degraded (5.293%); 0 B/s, 32 objects/s recovering 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: osdmap e127: 8 total, 8 up, 8 in 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9951/187995 objects degraded (5.293%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9951/187995 objects degraded (5.293%); 0 B/s, 32 objects/s recovering 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: osdmap e127: 8 total, 8 up, 8 in 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: failed to encode map e127 with expected crc 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:08.979 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:09.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: pgmap v397: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 9951/187995 objects degraded (5.293%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:09.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: pgmap v398: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 666 op/s; 9951/187995 objects degraded (5.293%); 0 B/s, 32 objects/s recovering 2024-04-05T03:07:09.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: failed to encode map e127 with expected crc 2024-04-05T03:07:09.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: failed to encode map e127 with expected crc 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: osdmap e127: 8 total, 8 up, 8 in 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: failed to encode map e127 with expected crc 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: failed to encode map e127 with expected crc 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:07:09.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:09.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:09 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:09.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:09 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:07:09.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:09 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/406585525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:09.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:09 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:09 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:07:09.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:09 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/406585525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:09 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:09 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-04-05T03:07:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:09 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/406585525' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:10 smithi150 ceph-mon[103445]: pgmap v400: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 9633/183972 objects degraded (5.236%); 0 B/s, 59 objects/s recovering 2024-04-05T03:07:10.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:10 smithi150 ceph-mon[107701]: pgmap v400: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 9633/183972 objects degraded (5.236%); 0 B/s, 59 objects/s recovering 2024-04-05T03:07:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:10 smithi196 ceph-mon[91352]: pgmap v400: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 9633/183972 objects degraded (5.236%); 0 B/s, 59 objects/s recovering 2024-04-05T03:07:11.878 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:07:11] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2024-04-05T03:07:11.878 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:11.878 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:12.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:12 smithi150 ceph-mon[103445]: pgmap v401: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9633/182178 objects degraded (5.288%); 0 B/s, 59 objects/s recovering 2024-04-05T03:07:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9633/182178 objects degraded (5.288%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:12 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2409075938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:12 smithi150 ceph-mon[107701]: pgmap v401: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9633/182178 objects degraded (5.288%); 0 B/s, 59 objects/s recovering 2024-04-05T03:07:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9633/182178 objects degraded (5.288%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:12.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:12 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2409075938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:12 smithi196 ceph-mon[91352]: pgmap v401: 89 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 84 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 9633/182178 objects degraded (5.288%); 0 B/s, 59 objects/s recovering 2024-04-05T03:07:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9633/182178 objects degraded (5.288%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:12 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2409075938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:13.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:13 smithi150 ceph-mon[107701]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 9493/173532 objects degraded (5.470%); 0 B/s, 66 objects/s recovering 2024-04-05T03:07:13.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:13 smithi150 ceph-mon[103445]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 9493/173532 objects degraded (5.470%); 0 B/s, 66 objects/s recovering 2024-04-05T03:07:14.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:13 smithi196 ceph-mon[91352]: pgmap v402: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s; 9493/173532 objects degraded (5.470%); 0 B/s, 66 objects/s recovering 2024-04-05T03:07:15.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:14 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/708308247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:15.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:14 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/708308247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:15.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:14 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/708308247' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:16.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:15 smithi196 ceph-mon[91352]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 9493/171426 objects degraded (5.538%); 0 B/s, 57 objects/s recovering 2024-04-05T03:07:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:15 smithi150 ceph-mon[107701]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 9493/171426 objects degraded (5.538%); 0 B/s, 57 objects/s recovering 2024-04-05T03:07:16.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:15 smithi150 ceph-mon[103445]: pgmap v403: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 9493/171426 objects degraded (5.538%); 0 B/s, 57 objects/s recovering 2024-04-05T03:07:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9493/171426 objects degraded (5.538%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:17.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9493/171426 objects degraded (5.538%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9493/171426 objects degraded (5.538%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:18 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1720984073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:18 smithi150 ceph-mon[103445]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 9493/167322 objects degraded (5.673%); 0 B/s, 45 objects/s recovering 2024-04-05T03:07:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:18 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1720984073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:18.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:18 smithi150 ceph-mon[107701]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 9493/167322 objects degraded (5.673%); 0 B/s, 45 objects/s recovering 2024-04-05T03:07:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:18 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1720984073' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:18 smithi196 ceph-mon[91352]: pgmap v404: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 9493/167322 objects degraded (5.673%); 0 B/s, 45 objects/s recovering 2024-04-05T03:07:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:20 smithi196 ceph-mon[91352]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9351/162390 objects degraded (5.758%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:20.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:20 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1129197067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:20.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:20 smithi150 ceph-mon[107701]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9351/162390 objects degraded (5.758%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:20.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:20 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1129197067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:20.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:20 smithi150 ceph-mon[103445]: pgmap v405: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 9351/162390 objects degraded (5.758%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:20.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:20 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1129197067' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:21.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:07:21] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2024-04-05T03:07:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-mon[103445]: pgmap v406: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 9351/160191 objects degraded (5.837%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9351/160191 objects degraded (5.837%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3612384891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:22.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:07:22.750+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:22.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-mon[107701]: pgmap v406: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 9351/160191 objects degraded (5.837%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:22.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9351/160191 objects degraded (5.837%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:22.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3612384891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:23.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:22 smithi196 ceph-mon[91352]: pgmap v406: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 659 op/s; 9351/160191 objects degraded (5.837%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:23.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9351/160191 objects degraded (5.837%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:23.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3612384891' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:23.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:23 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:23.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:23 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:24.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:23 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:24 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:24 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:24 smithi150 ceph-mon[107701]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 9208/151008 objects degraded (6.098%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:24 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:24 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:24.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:24 smithi150 ceph-mon[103445]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 9208/151008 objects degraded (6.098%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:25.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:24 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:25.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:24 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:25.137 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:24 smithi196 ceph-mon[91352]: pgmap v407: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 9208/151008 objects degraded (6.098%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:25.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:25 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1556103467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:25.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:25 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1556103467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:26.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:25 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1556103467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:26.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:26 smithi150 ceph-mon[107701]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9208/148908 objects degraded (6.184%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:26.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:26 smithi150 ceph-mon[103445]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9208/148908 objects degraded (6.184%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:27.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:26 smithi196 ceph-mon[91352]: pgmap v408: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9208/148908 objects degraded (6.184%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:27.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:27.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9208/148908 objects degraded (6.184%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:27.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:27 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1600093735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9208/148908 objects degraded (6.184%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:27.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:27 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1600093735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:28.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9208/148908 objects degraded (6.184%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:28.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:27 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1600093735' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:28 smithi150 ceph-mon[103445]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 9208/144741 objects degraded (6.362%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:28.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:28 smithi150 ceph-mon[107701]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 9208/144741 objects degraded (6.362%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:29.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:28 smithi196 ceph-mon[91352]: pgmap v409: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 9208/144741 objects degraded (6.362%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:29.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:29 smithi150 ceph-mon[107701]: pgmap v410: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 9063/139701 objects degraded (6.487%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:29.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:29 smithi150 ceph-mon[103445]: pgmap v410: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 9063/139701 objects degraded (6.487%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:30.037 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:29 smithi196 ceph-mon[91352]: pgmap v410: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 9063/139701 objects degraded (6.487%); 0 B/s, 35 objects/s recovering 2024-04-05T03:07:30.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:30 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2089204410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:30.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:30 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2089204410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:31.051 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:30 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2089204410' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:31.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:31 smithi150 ceph-mon[107701]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 9063/137532 objects degraded (6.590%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:31.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:07:31] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2024-04-05T03:07:31.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:31 smithi150 ceph-mon[103445]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 9063/137532 objects degraded (6.590%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:32.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:31 smithi196 ceph-mon[91352]: pgmap v411: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 9063/137532 objects degraded (6.590%); 0 B/s, 23 objects/s recovering 2024-04-05T03:07:32.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 9063/137532 objects degraded (6.590%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:32.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 9063/137532 objects degraded (6.590%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:33.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 9063/137532 objects degraded (6.590%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:33.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:33 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4119858765' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:33.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:33 smithi150 ceph-mon[107701]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 8903/128388 objects degraded (6.934%); 0 B/s, 37 objects/s recovering 2024-04-05T03:07:33.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:33 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4119858765' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:33.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:33 smithi150 ceph-mon[103445]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 8903/128388 objects degraded (6.934%); 0 B/s, 37 objects/s recovering 2024-04-05T03:07:34.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:33 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4119858765' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:34.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:33 smithi196 ceph-mon[91352]: pgmap v412: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 8903/128388 objects degraded (6.934%); 0 B/s, 37 objects/s recovering 2024-04-05T03:07:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3866464839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:35.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3866464839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:35.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3866464839' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:36.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:36 smithi196 ceph-mon[91352]: pgmap v413: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 8903/126315 objects degraded (7.048%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:36.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:36 smithi150 ceph-mon[103445]: pgmap v413: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 8903/126315 objects degraded (7.048%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:36.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:36 smithi150 ceph-mon[107701]: pgmap v413: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 685 op/s; 8903/126315 objects degraded (7.048%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:37.768 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8903/126315 objects degraded (7.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:37.769 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8903/126315 objects degraded (7.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8903/126315 objects degraded (7.048%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:38.227 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:37 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:07:37.768+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:38 smithi196 ceph-mon[91352]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 8903/122085 objects degraded (7.292%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:38 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:38 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2312813858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:38 smithi150 ceph-mon[103445]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 8903/122085 objects degraded (7.292%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:38 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:38 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2312813858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:38 smithi150 ceph-mon[107701]: pgmap v414: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 8903/122085 objects degraded (7.292%); 0 B/s, 25 objects/s recovering 2024-04-05T03:07:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:38 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:38.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:38 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2312813858' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:39 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:39 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:39 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:39.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:39 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:39 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:39.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:39 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:40.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:40 smithi196 ceph-mon[91352]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 8736/117150 objects degraded (7.457%); 0 B/s, 39 objects/s recovering 2024-04-05T03:07:40.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:40 smithi150 ceph-mon[103445]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 8736/117150 objects degraded (7.457%); 0 B/s, 39 objects/s recovering 2024-04-05T03:07:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:40 smithi150 ceph-mon[107701]: pgmap v415: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 766 op/s; 8736/117150 objects degraded (7.457%); 0 B/s, 39 objects/s recovering 2024-04-05T03:07:41.878 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3533896399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:41.878 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:41.879 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:07:41] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-05T03:07:41.879 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3533896399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:41.879 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3533896399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:42.863 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:42 smithi196 ceph-mon[91352]: pgmap v416: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8736/114957 objects degraded (7.599%); 0 B/s, 27 objects/s recovering 2024-04-05T03:07:42.863 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:42 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8736/117150 objects degraded (7.457%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:42 smithi150 ceph-mon[103445]: pgmap v416: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8736/114957 objects degraded (7.599%); 0 B/s, 27 objects/s recovering 2024-04-05T03:07:42.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:42 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8736/117150 objects degraded (7.457%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:42.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:42 smithi150 ceph-mon[107701]: pgmap v416: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 8736/114957 objects degraded (7.599%); 0 B/s, 27 objects/s recovering 2024-04-05T03:07:42.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:42 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8736/117150 objects degraded (7.457%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:43.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:43 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3943250012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:43.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:43 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3943250012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:43.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:43 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3943250012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:44.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:44 smithi196 ceph-mon[91352]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8563/105597 objects degraded (8.109%); 0 B/s, 41 objects/s recovering 2024-04-05T03:07:44.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:44 smithi150 ceph-mon[103445]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8563/105597 objects degraded (8.109%); 0 B/s, 41 objects/s recovering 2024-04-05T03:07:44.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:44 smithi150 ceph-mon[107701]: pgmap v417: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 8563/105597 objects degraded (8.109%); 0 B/s, 41 objects/s recovering 2024-04-05T03:07:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:46 smithi150 ceph-mon[103445]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 8563/103395 objects degraded (8.282%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:46.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:46 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1622151753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:46 smithi150 ceph-mon[107701]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 8563/103395 objects degraded (8.282%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:46.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:46 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1622151753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:47.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:46 smithi196 ceph-mon[91352]: pgmap v418: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 8563/103395 objects degraded (8.282%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:47.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:46 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1622151753' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:47 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8563/103395 objects degraded (8.282%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:47.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:47 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8563/103395 objects degraded (8.282%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:47 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8563/103395 objects degraded (8.282%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:48.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:48 smithi150 ceph-mon[103445]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8563/99006 objects degraded (8.649%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:48.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:48 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2161108158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:48.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:48 smithi150 ceph-mon[107701]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8563/99006 objects degraded (8.649%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:48.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:48 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2161108158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:49.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:48 smithi196 ceph-mon[91352]: pgmap v419: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 8563/99006 objects degraded (8.649%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:49.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:48 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2161108158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:50.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:50 smithi150 ceph-mon[107701]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 8386/94008 objects degraded (8.921%); 0 B/s, 43 objects/s recovering 2024-04-05T03:07:50.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:50 smithi150 ceph-mon[103445]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 8386/94008 objects degraded (8.921%); 0 B/s, 43 objects/s recovering 2024-04-05T03:07:51.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:50 smithi196 ceph-mon[91352]: pgmap v420: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 8386/94008 objects degraded (8.921%); 0 B/s, 43 objects/s recovering 2024-04-05T03:07:51.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:51 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3474212034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:07:51] "GET /metrics HTTP/1.1" 200 36485 "" "Prometheus/2.43.0" 2024-04-05T03:07:51.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:51 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3474212034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:52.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:51 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3474212034' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:52.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:52 smithi150 ceph-mon[103445]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 8386/91884 objects degraded (9.127%); 0 B/s, 29 objects/s recovering 2024-04-05T03:07:52.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:52 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8386/91884 objects degraded (9.127%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:52.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:07:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:07:52.785+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:52.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:52 smithi150 ceph-mon[107701]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 8386/91884 objects degraded (9.127%); 0 B/s, 29 objects/s recovering 2024-04-05T03:07:52.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:52 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8386/91884 objects degraded (9.127%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:53.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:52 smithi196 ceph-mon[91352]: pgmap v421: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.9 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 8386/91884 objects degraded (9.127%); 0 B/s, 29 objects/s recovering 2024-04-05T03:07:53.138 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:52 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8386/91884 objects degraded (9.127%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:53.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:53 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:53.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:53 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4089243666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:53 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:53.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:53 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4089243666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:54.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:53 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:54.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:53 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4089243666' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:54 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:54 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:54.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:54 smithi150 ceph-mon[103445]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 8217/82575 objects degraded (9.951%); 0 B/s, 43 objects/s recovering 2024-04-05T03:07:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:54 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:54 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:54.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:54 smithi150 ceph-mon[107701]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 8217/82575 objects degraded (9.951%); 0 B/s, 43 objects/s recovering 2024-04-05T03:07:55.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:54 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:07:55.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:54 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:07:55.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:54 smithi196 ceph-mon[91352]: pgmap v422: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.7 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 8217/82575 objects degraded (9.951%); 0 B/s, 43 objects/s recovering 2024-04-05T03:07:55.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:55 smithi150 ceph-mon[103445]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 8217/80577 objects degraded (10.198%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:55.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:55 smithi150 ceph-mon[107701]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 8217/80577 objects degraded (10.198%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:56.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:55 smithi196 ceph-mon[91352]: pgmap v423: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 8217/80577 objects degraded (10.198%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:56.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:56 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4132731780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:56.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:56.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:56 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4132731780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:57.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:07:57.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:56 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4132731780' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:57 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8217/80577 objects degraded (10.198%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:57.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:57 smithi150 ceph-mon[103445]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8217/76347 objects degraded (10.763%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:57.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:57 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8217/80577 objects degraded (10.198%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:57.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:57 smithi150 ceph-mon[107701]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8217/76347 objects degraded (10.763%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:58.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:57 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8217/80577 objects degraded (10.198%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:07:58.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:57 smithi196 ceph-mon[91352]: pgmap v424: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 8217/76347 objects degraded (10.763%); 0 B/s, 28 objects/s recovering 2024-04-05T03:07:59.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/982268879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:59.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/982268879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:07:59.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/982268879' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:00.037 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:07:59 smithi196 ceph-mon[91352]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 8027/71319 objects degraded (11.255%); 0 B/s, 44 objects/s recovering 2024-04-05T03:08:00.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:07:59 smithi150 ceph-mon[107701]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 8027/71319 objects degraded (11.255%); 0 B/s, 44 objects/s recovering 2024-04-05T03:08:00.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:07:59 smithi150 ceph-mon[103445]: pgmap v425: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 8027/71319 objects degraded (11.255%); 0 B/s, 44 objects/s recovering 2024-04-05T03:08:01.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:01 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4125059223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:01.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:01 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4125059223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:01.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:08:01] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-04-05T03:08:01.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:01 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4125059223' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:02.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:02 smithi196 ceph-mon[91352]: pgmap v426: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 8027/69186 objects degraded (11.602%); 0 B/s, 29 objects/s recovering 2024-04-05T03:08:02.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 8027/71319 objects degraded (11.255%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:02.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:02 smithi150 ceph-mon[103445]: pgmap v426: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 8027/69186 objects degraded (11.602%); 0 B/s, 29 objects/s recovering 2024-04-05T03:08:02.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 8027/71319 objects degraded (11.255%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:02.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:02 smithi150 ceph-mon[107701]: pgmap v426: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 8027/69186 objects degraded (11.602%); 0 B/s, 29 objects/s recovering 2024-04-05T03:08:02.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 8027/71319 objects degraded (11.255%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:04 smithi150 ceph-mon[103445]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 7822/59985 objects degraded (13.040%); 0 B/s, 46 objects/s recovering 2024-04-05T03:08:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:04 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4183987414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:04.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:04 smithi150 ceph-mon[107701]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 7822/59985 objects degraded (13.040%); 0 B/s, 46 objects/s recovering 2024-04-05T03:08:04.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:04 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4183987414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:04 smithi196 ceph-mon[91352]: pgmap v427: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s; 7822/59985 objects degraded (13.040%); 0 B/s, 46 objects/s recovering 2024-04-05T03:08:05.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:04 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4183987414' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:06.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:06 smithi150 ceph-mon[103445]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 7822/57846 objects degraded (13.522%); 0 B/s, 32 objects/s recovering 2024-04-05T03:08:06.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:06 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/151770161' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:06.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:06 smithi150 ceph-mon[107701]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 7822/57846 objects degraded (13.522%); 0 B/s, 32 objects/s recovering 2024-04-05T03:08:06.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:06 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/151770161' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:07.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:06 smithi196 ceph-mon[91352]: pgmap v428: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.2 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 7822/57846 objects degraded (13.522%); 0 B/s, 32 objects/s recovering 2024-04-05T03:08:07.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:06 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/151770161' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:07.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:07 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7822/57846 objects degraded (13.522%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:07.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:07 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7822/57846 objects degraded (13.522%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:08.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:07 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7822/57846 objects degraded (13.522%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:08.928 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:08 smithi196 ceph-mon[91352]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 7822/53532 objects degraded (14.612%); 0 B/s, 32 objects/s recovering 2024-04-05T03:08:08.928 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:08 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:08.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:08 smithi150 ceph-mon[107701]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 7822/53532 objects degraded (14.612%); 0 B/s, 32 objects/s recovering 2024-04-05T03:08:08.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:08 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:08 smithi150 ceph-mon[103445]: pgmap v429: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1.1 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 7822/53532 objects degraded (14.612%); 0 B/s, 32 objects/s recovering 2024-04-05T03:08:08.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:08 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:09.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:09 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4107992114' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:09.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:09 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4107992114' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:10.015 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:09 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4107992114' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:10.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:10 smithi150 ceph-mon[103445]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1011 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7620/48579 objects degraded (15.686%); 0 B/s, 49 objects/s recovering 2024-04-05T03:08:10.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:10 smithi150 ceph-mon[107701]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1011 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7620/48579 objects degraded (15.686%); 0 B/s, 49 objects/s recovering 2024-04-05T03:08:11.105 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:10 smithi196 ceph-mon[91352]: pgmap v430: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 1011 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 7620/48579 objects degraded (15.686%); 0 B/s, 49 objects/s recovering 2024-04-05T03:08:11.712 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:11.713 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:11 smithi196 ceph-mon[91352]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 968 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 7620/46512 objects degraded (16.383%); 0 B/s, 33 objects/s recovering 2024-04-05T03:08:11.713 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:11 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3579932455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:11.821 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:11.821 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-mon[103445]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 968 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 7620/46512 objects degraded (16.383%); 0 B/s, 33 objects/s recovering 2024-04-05T03:08:11.821 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3579932455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:11.821 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:11.821 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-mon[107701]: pgmap v431: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 968 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 7620/46512 objects degraded (16.383%); 0 B/s, 33 objects/s recovering 2024-04-05T03:08:11.822 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3579932455' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:11.822 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:08:11] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-04-05T03:08:12.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:12 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7620/46512 objects degraded (16.383%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:12.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:12 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7620/46512 objects degraded (16.383%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:12 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7620/46512 objects degraded (16.383%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[103445]: pgmap v432: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 786 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 7413/37776 objects degraded (19.624%); 0 B/s, 51 objects/s recovering 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:08:14.460+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:14 smithi150 ceph-mon[107701]: pgmap v432: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 786 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 7413/37776 objects degraded (19.624%); 0 B/s, 51 objects/s recovering 2024-04-05T03:08:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:14.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:14 smithi196 ceph-mon[91352]: pgmap v432: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 786 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 7413/37776 objects degraded (19.624%); 0 B/s, 51 objects/s recovering 2024-04-05T03:08:15.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3228152739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: pgmap v433: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 786 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 7413/37776 objects degraded (19.624%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:15.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:15 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3228152739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: pgmap v433: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 786 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 7413/37776 objects degraded (19.624%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:15.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3228152739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: pgmap v433: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 786 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 692 op/s; 7413/37776 objects degraded (19.624%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:15.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:15 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:17 smithi196 ceph-mon[91352]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 746 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 7413/35838 objects degraded (20.685%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2151111680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:17 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7413/35838 objects degraded (20.685%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:17 smithi196 ceph-mon[91352]: osdmap e128: 8 total, 8 up, 8 in 2024-04-05T03:08:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:17 smithi196 ceph-mon[91352]: failed to encode map e128 with expected crc 2024-04-05T03:08:17.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:17 smithi196 ceph-mon[91352]: failed to encode map e128 with expected crc 2024-04-05T03:08:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[103445]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 746 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 7413/35838 objects degraded (20.685%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2151111680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7413/35838 objects degraded (20.685%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[103445]: osdmap e128: 8 total, 8 up, 8 in 2024-04-05T03:08:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[103445]: failed to encode map e128 with expected crc 2024-04-05T03:08:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[103445]: failed to encode map e128 with expected crc 2024-04-05T03:08:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[107701]: pgmap v434: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 746 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 7413/35838 objects degraded (20.685%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2151111680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7413/35838 objects degraded (20.685%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[107701]: osdmap e128: 8 total, 8 up, 8 in 2024-04-05T03:08:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[107701]: failed to encode map e128 with expected crc 2024-04-05T03:08:17.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:17 smithi150 ceph-mon[107701]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:18 smithi196 ceph-mon[91352]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:18 smithi196 ceph-mon[91352]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:18 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:18 smithi196 ceph-mon[91352]: osdmap e129: 8 total, 8 up, 8 in 2024-04-05T03:08:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:18 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[103445]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[103445]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[103445]: osdmap e129: 8 total, 8 up, 8 in 2024-04-05T03:08:18.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:18.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[107701]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[107701]: failed to encode map e128 with expected crc 2024-04-05T03:08:18.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:18.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[107701]: osdmap e129: 8 total, 8 up, 8 in 2024-04-05T03:08:18.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:18 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:19 smithi196 ceph-mon[91352]: pgmap v437: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 624 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 7413/29994 objects degraded (24.715%); 0 B/s, 30 objects/s recovering 2024-04-05T03:08:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[103445]: pgmap v437: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 624 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 7413/29994 objects degraded (24.715%); 0 B/s, 30 objects/s recovering 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: failed to encode map e129 with expected crc 2024-04-05T03:08:19.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:19 smithi150 ceph-mon[107701]: pgmap v437: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 624 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 7413/29994 objects degraded (24.715%); 0 B/s, 30 objects/s recovering 2024-04-05T03:08:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:20 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2971785471' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:20.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:20 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2971785471' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:20.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:20 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2971785471' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:21 smithi196 ceph-mon[91352]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 534 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7132/25674 objects degraded (27.779%); 0 B/s, 42 objects/s recovering 2024-04-05T03:08:21.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:21 smithi150 ceph-mon[103445]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 534 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7132/25674 objects degraded (27.779%); 0 B/s, 42 objects/s recovering 2024-04-05T03:08:21.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:21 smithi150 ceph-mon[107701]: pgmap v438: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 534 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7132/25674 objects degraded (27.779%); 0 B/s, 42 objects/s recovering 2024-04-05T03:08:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:08:21] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-04-05T03:08:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3092973769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:22 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7132/25674 objects degraded (27.779%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:22.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3092973769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:22.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:22 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7132/25674 objects degraded (27.779%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:22.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3092973769' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:22.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:22 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7132/25674 objects degraded (27.779%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:23.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:23 smithi196 ceph-mon[91352]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 463 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7132/22272 objects degraded (32.022%); 0 B/s, 35 objects/s recovering 2024-04-05T03:08:23.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:23 smithi150 ceph-mon[103445]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 463 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7132/22272 objects degraded (32.022%); 0 B/s, 35 objects/s recovering 2024-04-05T03:08:23.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:23 smithi150 ceph-mon[107701]: pgmap v439: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 463 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 7132/22272 objects degraded (32.022%); 0 B/s, 35 objects/s recovering 2024-04-05T03:08:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:25 smithi196 ceph-mon[91352]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 7021/17337 objects degraded (40.497%); 0 B/s, 48 objects/s recovering 2024-04-05T03:08:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:25 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3432027669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:25 smithi150 ceph-mon[103445]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 7021/17337 objects degraded (40.497%); 0 B/s, 48 objects/s recovering 2024-04-05T03:08:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:25 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3432027669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:25 smithi150 ceph-mon[107701]: pgmap v440: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 360 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 7021/17337 objects degraded (40.497%); 0 B/s, 48 objects/s recovering 2024-04-05T03:08:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:25 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3432027669' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:26.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:27.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:27 smithi150 ceph-mon[107701]: pgmap v441: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 315 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7021/15150 objects degraded (46.343%); 0 B/s, 42 objects/s recovering 2024-04-05T03:08:27.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:27 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 7021/15150 objects degraded (46.343%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:27.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:27 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2908134456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:27 smithi150 ceph-mon[103445]: pgmap v441: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 315 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7021/15150 objects degraded (46.343%); 0 B/s, 42 objects/s recovering 2024-04-05T03:08:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:27 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 7021/15150 objects degraded (46.343%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:27 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2908134456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:27 smithi196 ceph-mon[91352]: pgmap v441: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 315 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 7021/15150 objects degraded (46.343%); 0 B/s, 42 objects/s recovering 2024-04-05T03:08:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:27 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 7021/15150 objects degraded (46.343%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:27.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:27 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2908134456' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:29.507 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Removed 111255 objects 2024-04-05T03:08:29.507 INFO:tasks.radosbench.radosbench.0.smithi150.stdout:Clean up completed and total clean up time :148.609 2024-04-05T03:08:29.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:29 smithi150 ceph-mon[103445]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 175 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 7021/8457 objects degraded (83.020%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:29 smithi150 ceph-mon[107701]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 175 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 7021/8457 objects degraded (83.020%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:29.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:08:29.477+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:29.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:29 smithi196 ceph-mon[91352]: pgmap v442: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 175 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 7021/8457 objects degraded (83.020%); 0 B/s, 38 objects/s recovering 2024-04-05T03:08:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:30.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3279711349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:30.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:30.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:30 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3279711349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:30.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:30 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:30.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:30 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:30.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:30 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-05T03:08:30.788 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:30 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3279711349' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:31 smithi150 ceph-mon[103445]: pgmap v443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 74 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 6891/3600 objects degraded (191.417%); 0 B/s, 43 objects/s recovering 2024-04-05T03:08:31.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:31 smithi150 ceph-mon[107701]: pgmap v443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 74 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 6891/3600 objects degraded (191.417%); 0 B/s, 43 objects/s recovering 2024-04-05T03:08:31.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:08:31] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2024-04-05T03:08:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:31 smithi196 ceph-mon[91352]: pgmap v443: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 74 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 6891/3600 objects degraded (191.417%); 0 B/s, 43 objects/s recovering 2024-04-05T03:08:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:32 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2027747970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:32.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:32 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 6891/3600 objects degraded (191.417%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:32.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:32 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2027747970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:32.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:32 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 6891/3600 objects degraded (191.417%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:32.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:32 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2027747970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:32.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:32 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 6891/3600 objects degraded (191.417%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:33 smithi150 ceph-mon[103445]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 25 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 6891/1251 objects degraded (550.839%); 0 B/s, 20 objects/s recovering 2024-04-05T03:08:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:33 smithi150 ceph-mon[107701]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 25 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 6891/1251 objects degraded (550.839%); 0 B/s, 20 objects/s recovering 2024-04-05T03:08:33.885 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-04-05T03:08:33.886 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-04-05T03:08:33.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:33 smithi196 ceph-mon[91352]: pgmap v444: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 25 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 6891/1251 objects degraded (550.839%); 0 B/s, 20 objects/s recovering 2024-04-05T03:08:34.235 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:08:35.669 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:35 smithi150 ceph-mon[103445]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6103/72 objects degraded (8476.389%); 0 B/s, 85 objects/s recovering 2024-04-05T03:08:35.669 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1884809767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:35.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:35 smithi150 ceph-mon[107701]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6103/72 objects degraded (8476.389%); 0 B/s, 85 objects/s recovering 2024-04-05T03:08:35.670 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1884809767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:35 smithi196 ceph-mon[91352]: pgmap v445: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 6103/72 objects degraded (8476.389%); 0 B/s, 85 objects/s recovering 2024-04-05T03:08:35.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1884809767' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:37.110 INFO:teuthology.orchestra.run.smithi150.stderr:pool 'unique_pool_3' removed 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 6103/72 objects degraded (8476.389%); 0 B/s, 76 objects/s recovering 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3827881867' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4120951041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 6103/72 objects degraded (8476.389%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T03:08:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: failed to encode map e130 with expected crc 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[103445]: osdmap e130: 8 total, 8 up, 8 in 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 6103/72 objects degraded (8476.389%); 0 B/s, 76 objects/s recovering 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3827881867' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4120951041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 6103/72 objects degraded (8476.389%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: failed to encode map e130 with expected crc 2024-04-05T03:08:37.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:37 smithi150 ceph-mon[107701]: osdmap e130: 8 total, 8 up, 8 in 2024-04-05T03:08:37.792 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-05T03:08:37.792 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-04-05T03:08:37.792 DEBUG:teuthology.parallel:result is None 2024-04-05T03:08:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: pgmap v446: 89 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 86 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 479 op/s; 6103/72 objects degraded (8476.389%); 0 B/s, 76 objects/s recovering 2024-04-05T03:08:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3827881867' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:08:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-05T03:08:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4120951041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:37.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 6103/72 objects degraded (8476.389%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-04-05T03:08:37.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-04-05T03:08:37.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: failed to encode map e130 with expected crc 2024-04-05T03:08:37.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:37 smithi196 ceph-mon[91352]: osdmap e130: 8 total, 8 up, 8 in 2024-04-05T03:08:38.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:38 smithi150 ceph-mon[103445]: failed to encode map e130 with expected crc 2024-04-05T03:08:38.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:38 smithi150 ceph-mon[103445]: failed to encode map e130 with expected crc 2024-04-05T03:08:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:38 smithi150 ceph-mon[107701]: failed to encode map e130 with expected crc 2024-04-05T03:08:38.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:38 smithi150 ceph-mon[107701]: failed to encode map e130 with expected crc 2024-04-05T03:08:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:38 smithi196 ceph-mon[91352]: failed to encode map e130 with expected crc 2024-04-05T03:08:38.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:38 smithi196 ceph-mon[91352]: failed to encode map e130 with expected crc 2024-04-05T03:08:39.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:39 smithi150 ceph-mon[103445]: pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:39.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:39 smithi150 ceph-mon[103445]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6103/72 objects degraded (8476.389%), 3 pgs degraded, 3 pgs undersized) 2024-04-05T03:08:39.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:39 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3489478405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:39 smithi150 ceph-mon[107701]: pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:39 smithi150 ceph-mon[107701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6103/72 objects degraded (8476.389%), 3 pgs degraded, 3 pgs undersized) 2024-04-05T03:08:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:39 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3489478405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:39 smithi196 ceph-mon[91352]: pgmap v448: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:39 smithi196 ceph-mon[91352]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6103/72 objects degraded (8476.389%), 3 pgs degraded, 3 pgs undersized) 2024-04-05T03:08:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:39 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3489478405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:41.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:08:41] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-04-05T03:08:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[103445]: pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:41.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1017546444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[107701]: pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:41.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1017546444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:41 smithi196 ceph-mon[91352]: pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:41 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:41.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1017546444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:43.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:43 smithi150 ceph-mon[103445]: pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:43.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:43 smithi150 ceph-mon[107701]: pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:43.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:43 smithi196 ceph-mon[91352]: pgmap v450: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:44.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:44 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2539010766' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:44.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:44 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2539010766' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:44.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:44 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2539010766' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[103445]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[103445]: Upgrade: osd.2 is safe to restart 2024-04-05T03:08:45.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[107701]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:45.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:45.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:45.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:45 smithi150 ceph-mon[107701]: Upgrade: osd.2 is safe to restart 2024-04-05T03:08:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:45 smithi196 ceph-mon[91352]: pgmap v451: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:45 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:45 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-05T03:08:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:45 smithi196 ceph-mon[91352]: Upgrade: osd.2 is safe to restart 2024-04-05T03:08:47.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/852639940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: Upgrade: Updating osd.2 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: Deploying daemon osd.2 on smithi150 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[107701]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/852639940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:47.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: Upgrade: Updating osd.2 2024-04-05T03:08:47.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:47.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:08:47.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:47.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: Deploying daemon osd.2 on smithi150 2024-04-05T03:08:47.229 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:47 smithi150 ceph-mon[103445]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:47.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:47.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/852639940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:47.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: Upgrade: Updating osd.2 2024-04-05T03:08:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-05T03:08:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:08:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: Deploying daemon osd.2 on smithi150 2024-04-05T03:08:47.387 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:47 smithi196 ceph-mon[91352]: pgmap v452: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:49.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-mon[103445]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:49.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4191812506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:49.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-mon[107701]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:49.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4191812506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:49.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:49 smithi196 ceph-mon[91352]: pgmap v453: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:49.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:49 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4191812506' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:49.858 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:49 smithi150 systemd[1]: Stopping Ceph osd.2 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:08:50.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[58685]: 2024-04-05T03:08:49.857+0000 7f53f26f7700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:08:50.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[58685]: 2024-04-05T03:08:49.857+0000 7f53f26f7700 -1 osd.2 130 *** Got signal Terminated *** 2024-04-05T03:08:50.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[58685]: 2024-04-05T03:08:49.857+0000 7f53f26f7700 -1 osd.2 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:08:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:50 smithi196 ceph-mon[91352]: osd.2 marked itself down and dead 2024-04-05T03:08:50.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:50 smithi150 ceph-mon[103445]: osd.2 marked itself down and dead 2024-04-05T03:08:50.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:50 smithi150 ceph-mon[107701]: osd.2 marked itself down and dead 2024-04-05T03:08:51.197 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:50 smithi150 podman[135925]: 2024-04-05 03:08:50.898862288 +0000 UTC m=+1.147111580 container died fee7e3893826bff6933dba1b87beed10adc04d362c5b92fd99063ad7866b90cf (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-05T03:08:51.477 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:51 smithi150 podman[135925]: 2024-04-05 03:08:51.288518822 +0000 UTC m=+1.536768116 container cleanup fee7e3893826bff6933dba1b87beed10adc04d362c5b92fd99063ad7866b90cf (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-04-05T03:08:51.477 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:51 smithi150 bash[135925]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2 2024-04-05T03:08:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:51.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: osdmap e131: 8 total, 7 up, 8 in 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1489580657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: osdmap e131: 8 total, 7 up, 8 in 2024-04-05T03:08:51.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1489580657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:51.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: pgmap v454: 73 pgs: 73 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: osdmap e131: 8 total, 7 up, 8 in 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: failed to encode map e131 with expected crc 2024-04-05T03:08:51.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:51 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1489580657' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:51.936 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:08:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:08:51] "GET /metrics HTTP/1.1" 200 34124 "" "Prometheus/2.43.0" 2024-04-05T03:08:51.937 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:51 smithi150 podman[135975]: 2024-04-05 03:08:51.508810905 +0000 UTC m=+0.602718675 container remove fee7e3893826bff6933dba1b87beed10adc04d362c5b92fd99063ad7866b90cf (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-04-05T03:08:52.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:51 smithi150 podman[136074]: 2024-04-05 03:08:51.840594109 +0000 UTC m=+0.018010177 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:52.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:51 smithi150 podman[136074]: 2024-04-05 03:08:51.949674329 +0000 UTC m=+0.127090430 container create 1893c90a4472ea127dee1fdc2556f50b68db0f0b3a639da5bf12a1260741f791 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:08:52.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:52 smithi150 podman[136074]: 2024-04-05 03:08:52.162654673 +0000 UTC m=+0.340070728 container init 1893c90a4472ea127dee1fdc2556f50b68db0f0b3a639da5bf12a1260741f791 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.vendor=CentOS) 2024-04-05T03:08:52.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:52 smithi150 podman[136074]: 2024-04-05 03:08:52.168161338 +0000 UTC m=+0.345577394 container start 1893c90a4472ea127dee1fdc2556f50b68db0f0b3a639da5bf12a1260741f791 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:08:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: osdmap e132: 8 total, 7 up, 8 in 2024-04-05T03:08:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:52 smithi196 ceph-mon[91352]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: osdmap e132: 8 total, 7 up, 8 in 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[103445]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:52 smithi150 podman[136074]: 2024-04-05 03:08:52.279220662 +0000 UTC m=+0.456636731 container attach 1893c90a4472ea127dee1fdc2556f50b68db0f0b3a639da5bf12a1260741f791 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-05T03:08:52.728 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:52 smithi150 podman[136074]: 2024-04-05 03:08:52.523120597 +0000 UTC m=+0.700536646 container died 1893c90a4472ea127dee1fdc2556f50b68db0f0b3a639da5bf12a1260741f791 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: osdmap e132: 8 total, 7 up, 8 in 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:52.729 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:52 smithi150 ceph-mon[107701]: failed to encode map e132 with expected crc 2024-04-05T03:08:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:53 smithi150 ceph-mon[103445]: pgmap v457: 73 pgs: 1 active+undersized+degraded, 9 active+undersized, 1 stale+active+clean, 62 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:08:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:53 smithi150 ceph-mon[103445]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:08:53.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:53 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2316559303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:53 smithi150 ceph-mon[107701]: pgmap v457: 73 pgs: 1 active+undersized+degraded, 9 active+undersized, 1 stale+active+clean, 62 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:08:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:53 smithi150 ceph-mon[107701]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:08:53.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:53 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2316559303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:53.478 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 podman[136120]: 2024-04-05 03:08:53.166441463 +0000 UTC m=+0.638021886 container remove 1893c90a4472ea127dee1fdc2556f50b68db0f0b3a639da5bf12a1260741f791 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-0028525, maintainer=Guillaume Abrioux ) 2024-04-05T03:08:53.479 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.2.service: Deactivated successfully. 2024-04-05T03:08:53.479 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 systemd[1]: Stopped Ceph osd.2 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:08:53.479 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.2.service: Consumed 27min 16.969s CPU time. 2024-04-05T03:08:53.479 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 systemd[1]: Starting Ceph osd.2 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:08:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:53 smithi196 ceph-mon[91352]: pgmap v457: 73 pgs: 1 active+undersized+degraded, 9 active+undersized, 1 stale+active+clean, 62 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:08:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:53 smithi196 ceph-mon[91352]: Health check failed: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:08:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:53 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2316559303' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:53.936 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 podman[136259]: 2024-04-05 03:08:53.522681104 +0000 UTC m=+0.017997119 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:53.936 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 podman[136259]: 2024-04-05 03:08:53.65146408 +0000 UTC m=+0.146780097 container create f87d9817fdf0d08dc734ae0eb9d4cd881dea8d72d445ce62124a4e6e34fde33f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-05T03:08:53.936 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 podman[136259]: 2024-04-05 03:08:53.854883634 +0000 UTC m=+0.350199650 container init f87d9817fdf0d08dc734ae0eb9d4cd881dea8d72d445ce62124a4e6e34fde33f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-04-05T03:08:53.936 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 podman[136259]: 2024-04-05 03:08:53.860908917 +0000 UTC m=+0.356224932 container start f87d9817fdf0d08dc734ae0eb9d4cd881dea8d72d445ce62124a4e6e34fde33f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-04-05T03:08:53.937 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:53 smithi150 podman[136259]: 2024-04-05 03:08:53.937304229 +0000 UTC m=+0.432620245 container attach f87d9817fdf0d08dc734ae0eb9d4cd881dea8d72d445ce62124a4e6e34fde33f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-05T03:08:54.632 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:08:54.633 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-05T03:08:54.633 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:08:54.633 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-05T03:08:54.633 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-05T03:08:54.634 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-05T03:08:54.634 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-05T03:08:54.634 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:08:54.634 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 bash[136259]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-05T03:08:54.634 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-05T03:08:54.635 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-05T03:08:54.635 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-05T03:08:54.636 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-05T03:08:54.636 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate[136274]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-05T03:08:54.636 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 podman[136259]: 2024-04-05 03:08:54.325470753 +0000 UTC m=+0.820786760 container died f87d9817fdf0d08dc734ae0eb9d4cd881dea8d72d445ce62124a4e6e34fde33f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-0028525, GIT_CLEAN=True, ceph=True) 2024-04-05T03:08:55.182 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:54 smithi150 podman[136411]: 2024-04-05 03:08:54.932978361 +0000 UTC m=+0.602319616 container remove f87d9817fdf0d08dc734ae0eb9d4cd881dea8d72d445ce62124a4e6e34fde33f (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:08:55.183 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 podman[136477]: 2024-04-05 03:08:55.047604406 +0000 UTC m=+0.018269753 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:08:55.183 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 podman[136477]: 2024-04-05 03:08:55.183927263 +0000 UTC m=+0.154592621 container create 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, RELEASE=squid-0028525, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-05T03:08:55.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:55 smithi150 ceph-mon[103445]: pgmap v458: 73 pgs: 1 active+undersized+degraded, 19 active+undersized, 53 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:08:55.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:55 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1908016675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:55.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:55 smithi150 ceph-mon[107701]: pgmap v458: 73 pgs: 1 active+undersized+degraded, 19 active+undersized, 53 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:08:55.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:55 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1908016675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:55.478 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 podman[136477]: 2024-04-05 03:08:55.403207133 +0000 UTC m=+0.373872481 container init 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-05T03:08:55.478 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 podman[136477]: 2024-04-05 03:08:55.409101118 +0000 UTC m=+0.379766466 container start 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-0028525, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-05T03:08:55.478 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 ceph-osd[136541]: -- 172.21.15.150:0/3477607921 <== mon.0 v2:172.21.15.150:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eff2313680 con 0x55eff22dd000 2024-04-05T03:08:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:55 smithi196 ceph-mon[91352]: pgmap v458: 73 pgs: 1 active+undersized+degraded, 19 active+undersized, 53 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:08:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:55 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1908016675' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:55.953 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 bash[136477]: 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb 2024-04-05T03:08:55.954 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 systemd[1]: Started Ceph osd.2 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:08:56.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:08:55 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[136515]: 2024-04-05T03:08:55.953+0000 7f94b96b97c0 -1 Falling back to public interface 2024-04-05T03:08:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:08:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:56.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:08:57.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:57 smithi150 ceph-mon[103445]: pgmap v459: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:08:57.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:57 smithi150 ceph-mon[107701]: pgmap v459: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:08:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:57 smithi196 ceph-mon[91352]: pgmap v459: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:08:58.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3519399444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:58.916 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3519399444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:58.916 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3519399444' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:08:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:59 smithi196 ceph-mon[91352]: pgmap v460: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:08:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:59 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:08:59 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:59 smithi150 ceph-mon[107701]: pgmap v460: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:08:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:59 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:08:59 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:59.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:59 smithi150 ceph-mon[103445]: pgmap v460: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:08:59.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:59 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:08:59.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:08:59 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:00.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:00 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1844150142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:00.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:00 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1844150142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:00 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1844150142' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:01.355 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[136515]: 2024-04-05T03:09:01.100+0000 7f94b96b97c0 -1 osd.2 130 log_to_monitors true 2024-04-05T03:09:01.605 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:09:01] "GET /metrics HTTP/1.1" 200 34119 "" "Prometheus/2.43.0" 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[103445]: pgmap v461: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[103445]: from='osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[103445]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[107701]: pgmap v461: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:01.606 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:01.607 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[107701]: from='osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:09:01.607 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:01 smithi150 ceph-mon[107701]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:09:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:01 smithi196 ceph-mon[91352]: pgmap v461: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:01 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:01 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:01 smithi196 ceph-mon[91352]: from='osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:09:01.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:01 smithi196 ceph-mon[91352]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-05T03:09:02.227 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[136515]: 2024-04-05T03:09:02.074+0000 7f94a6b09700 -1 osd.2 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: from='osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: failed to encode map e133 with expected crc 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: failed to encode map e133 with expected crc 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: osdmap e133: 8 total, 7 up, 8 in 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: from='osd.2 ' entity='osd.2' 2024-04-05T03:09:02.822 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2063682041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: from='osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: failed to encode map e133 with expected crc 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: failed to encode map e133 with expected crc 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: osdmap e133: 8 total, 7 up, 8 in 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: from='osd.2 ' entity='osd.2' 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:02.823 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:02 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2063682041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-05T03:09:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: from='osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: failed to encode map e133 with expected crc 2024-04-05T03:09:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: failed to encode map e133 with expected crc 2024-04-05T03:09:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: osdmap e133: 8 total, 7 up, 8 in 2024-04-05T03:09:02.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:02.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: from='osd.2 ' entity='osd.2' 2024-04-05T03:09:02.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:02.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:02 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2063682041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:03.792 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: pgmap v463: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:03.792 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350] boot 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: osdmap e134: 8 total, 8 up, 8 in 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[107701]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: pgmap v463: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350] boot 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:09:03.793 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.794 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.794 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.794 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: osdmap e134: 8 total, 8 up, 8 in 2024-04-05T03:09:03.794 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:03 smithi150 ceph-mon[103445]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: pgmap v463: 73 pgs: 2 active+undersized+degraded, 21 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: osd.2 [v2:172.21.15.150:6818/1032167350,v1:172.21.15.150:6819/1032167350] boot 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: failed to encode map e134 with expected crc 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: osdmap e134: 8 total, 8 up, 8 in 2024-04-05T03:09:03.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:03 smithi196 ceph-mon[91352]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:09:04.379+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: osdmap e135: 8 total, 8 up, 8 in 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:04.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.888 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: osdmap e135: 8 total, 8 up, 8 in 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:04.981 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e134 with expected crc 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: osdmap e135: 8 total, 8 up, 8 in 2024-04-05T03:09:04.982 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.983 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:04.984 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: failed to encode map e135 with expected crc 2024-04-05T03:09:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: pgmap v466: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:05.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: pgmap v467: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:05.887 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2285643432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: failed to encode map e135 with expected crc 2024-04-05T03:09:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: pgmap v466: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: pgmap v467: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2285643432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: failed to encode map e135 with expected crc 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: pgmap v466: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: pgmap v467: 73 pgs: 5 peering, 2 active+undersized+degraded, 16 active+undersized, 50 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:05.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2285643432' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:07.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:07 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:07.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:07 smithi196 ceph-mon[91352]: pgmap v468: 73 pgs: 5 peering, 1 active+undersized+degraded, 14 active+undersized, 53 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:09:07.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:07 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1718707729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:07.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:07 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:07.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:07 smithi150 ceph-mon[103445]: pgmap v468: 73 pgs: 5 peering, 1 active+undersized+degraded, 14 active+undersized, 53 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:09:07.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:07 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1718707729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:07.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:07 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:07.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:07 smithi150 ceph-mon[107701]: pgmap v468: 73 pgs: 5 peering, 1 active+undersized+degraded, 14 active+undersized, 53 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2/72 objects degraded (2.778%) 2024-04-05T03:09:07.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:07 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1718707729' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:08 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:09:08.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:08 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:09:08.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:08 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded (PG_DEGRADED) 2024-04-05T03:09:09.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:09 smithi196 ceph-mon[91352]: pgmap v469: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:09.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:09 smithi150 ceph-mon[107701]: pgmap v469: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:09.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:09 smithi150 ceph-mon[103445]: pgmap v469: 73 pgs: 5 peering, 68 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:10.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:10 smithi196 ceph-mon[91352]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-05T03:09:10.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1338760411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:10.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:10 smithi150 ceph-mon[103445]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-05T03:09:10.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1338760411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:10.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:10 smithi150 ceph-mon[107701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/72 objects degraded (2.778%), 1 pg degraded) 2024-04-05T03:09:10.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1338760411' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:11.378 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-mon[107701]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:11.378 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:11.378 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:11.379 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-mon[103445]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:11.379 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:11.379 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:11 smithi196 ceph-mon[91352]: pgmap v470: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:11 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:11.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:09:11] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:09:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:12 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3851165191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:12.394 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:12 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3851165191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:12.394 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:12 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3851165191' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:13.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:13 smithi196 ceph-mon[91352]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:13.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:13 smithi150 ceph-mon[107701]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:13.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:13 smithi150 ceph-mon[103445]: pgmap v471: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:14.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:14 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/741248053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:14 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/741248053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:14.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:14 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/741248053' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:15.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:15 smithi196 ceph-mon[91352]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:15.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:15 smithi150 ceph-mon[103445]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:15.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:15 smithi150 ceph-mon[107701]: pgmap v472: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:17.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:17 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:17.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:17 smithi196 ceph-mon[91352]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:17.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3323688984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:17.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:17 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:17.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:17 smithi150 ceph-mon[107701]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:17.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3323688984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:17.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:17 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:17.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:17 smithi150 ceph-mon[103445]: pgmap v473: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:17.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3323688984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:19 smithi196 ceph-mon[91352]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:19 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3521716555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:19 smithi150 ceph-mon[103445]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:19.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:19 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3521716555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:19.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:19 smithi150 ceph-mon[107701]: pgmap v474: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:19.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:19 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3521716555' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:20 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:20 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:20 smithi196 ceph-mon[91352]: Upgrade: osd.3 is safe to restart 2024-04-05T03:09:20.684 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:20 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:20.684 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:20 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:20.684 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:20 smithi150 ceph-mon[107701]: Upgrade: osd.3 is safe to restart 2024-04-05T03:09:20.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:20 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:20.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:20 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-05T03:09:20.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:20 smithi150 ceph-mon[103445]: Upgrade: osd.3 is safe to restart 2024-04-05T03:09:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[103445]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3879440984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:09:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[107701]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3879440984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:09:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:21 smithi196 ceph-mon[91352]: pgmap v475: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:21 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3879440984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:21.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:21 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:21.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-05T03:09:21.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:21.801 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:09:21] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:09:22.307 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:22 smithi150 ceph-mon[103445]: Upgrade: Updating osd.3 2024-04-05T03:09:22.307 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:22 smithi150 ceph-mon[103445]: Deploying daemon osd.3 on smithi150 2024-04-05T03:09:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:22 smithi196 ceph-mon[91352]: Upgrade: Updating osd.3 2024-04-05T03:09:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:22 smithi196 ceph-mon[91352]: Deploying daemon osd.3 on smithi150 2024-04-05T03:09:22.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:22 smithi150 ceph-mon[107701]: Upgrade: Updating osd.3 2024-04-05T03:09:22.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:22 smithi150 ceph-mon[107701]: Deploying daemon osd.3 on smithi150 2024-04-05T03:09:23.438 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:23 smithi150 ceph-mon[103445]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:23.438 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:23 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1673393490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:23.438 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:23 smithi150 ceph-mon[107701]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:23.438 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:23 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1673393490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:23.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:23 smithi196 ceph-mon[91352]: pgmap v476: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:23.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:23 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1673393490' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:24.977 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:24 smithi150 systemd[1]: Stopping Ceph osd.3 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:09:25.315 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[65322]: 2024-04-05T03:09:24.992+0000 7f705959d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:09:25.315 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[65322]: 2024-04-05T03:09:24.992+0000 7f705959d700 -1 osd.3 135 *** Got signal Terminated *** 2024-04-05T03:09:25.315 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[65322]: 2024-04-05T03:09:24.992+0000 7f705959d700 -1 osd.3 135 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:09:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:25 smithi196 ceph-mon[91352]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:25 smithi196 ceph-mon[91352]: osd.3 marked itself down and dead 2024-04-05T03:09:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:25 smithi150 ceph-mon[107701]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:25 smithi150 ceph-mon[107701]: osd.3 marked itself down and dead 2024-04-05T03:09:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:25 smithi150 ceph-mon[103445]: pgmap v477: 73 pgs: 73 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:25 smithi150 ceph-mon[103445]: osd.3 marked itself down and dead 2024-04-05T03:09:26.181 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:25 smithi150 podman[141180]: 2024-04-05 03:09:25.901440331 +0000 UTC m=+1.037459679 container died 5d887a680c09e24baecbfe20f282432dd49d3e29c588daf4e2dfc18b12abb9c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-05T03:09:26.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:09:26.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: osdmap e136: 8 total, 7 up, 8 in 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3615371160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:26.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[103445]: osdmap e137: 8 total, 7 up, 8 in 2024-04-05T03:09:26.479 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:26 smithi150 podman[141180]: 2024-04-05 03:09:26.373757878 +0000 UTC m=+1.509777223 container cleanup 5d887a680c09e24baecbfe20f282432dd49d3e29c588daf4e2dfc18b12abb9c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-6fda41f, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:09:26.479 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:26 smithi150 bash[141180]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3 2024-04-05T03:09:26.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:09:26.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: osdmap e136: 8 total, 7 up, 8 in 2024-04-05T03:09:26.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3615371160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:26.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:26 smithi150 ceph-mon[107701]: osdmap e137: 8 total, 7 up, 8 in 2024-04-05T03:09:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-05T03:09:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: osdmap e136: 8 total, 7 up, 8 in 2024-04-05T03:09:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: failed to encode map e136 with expected crc 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3615371160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:26.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:26 smithi196 ceph-mon[91352]: osdmap e137: 8 total, 7 up, 8 in 2024-04-05T03:09:26.977 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:26 smithi150 podman[141229]: 2024-04-05 03:09:26.717431354 +0000 UTC m=+0.813557136 container remove 5d887a680c09e24baecbfe20f282432dd49d3e29c588daf4e2dfc18b12abb9c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:09:27.278 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:27 smithi150 podman[141330]: 2024-04-05 03:09:26.9270802 +0000 UTC m=+0.020596361 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:27.279 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:27 smithi150 podman[141330]: 2024-04-05 03:09:27.035063447 +0000 UTC m=+0.128579592 container create af42714468d54eb117d74122533a4ab93e3828ed7d29a8dff7e27105a868b362 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-04-05T03:09:27.587 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: pgmap v479: 73 pgs: 12 stale+active+clean, 61 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:27.587 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.587 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.587 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[107701]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: pgmap v479: 73 pgs: 12 stale+active+clean, 61 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:27.588 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:27 smithi150 ceph-mon[103445]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.589 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:27 smithi150 podman[141330]: 2024-04-05 03:09:27.279856013 +0000 UTC m=+0.373372159 container init af42714468d54eb117d74122533a4ab93e3828ed7d29a8dff7e27105a868b362 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:09:27.589 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:27 smithi150 podman[141330]: 2024-04-05 03:09:27.285652018 +0000 UTC m=+0.379168165 container start af42714468d54eb117d74122533a4ab93e3828ed7d29a8dff7e27105a868b362 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-05T03:09:27.589 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:27 smithi150 podman[141330]: 2024-04-05 03:09:27.370935619 +0000 UTC m=+0.464451771 container attach af42714468d54eb117d74122533a4ab93e3828ed7d29a8dff7e27105a868b362 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS) 2024-04-05T03:09:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: pgmap v479: 73 pgs: 12 stale+active+clean, 61 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-04-05T03:09:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:27 smithi196 ceph-mon[91352]: failed to encode map e137 with expected crc 2024-04-05T03:09:27.890 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:27 smithi150 podman[141330]: 2024-04-05 03:09:27.589005181 +0000 UTC m=+0.682521333 container died af42714468d54eb117d74122533a4ab93e3828ed7d29a8dff7e27105a868b362 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2) 2024-04-05T03:09:28.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:28 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4143393608' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:28.688 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:28 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4143393608' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:28.689 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:28 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4143393608' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:28.689 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 podman[141379]: 2024-04-05 03:09:28.25026631 +0000 UTC m=+0.655580612 container remove af42714468d54eb117d74122533a4ab93e3828ed7d29a8dff7e27105a868b362 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-0028525) 2024-04-05T03:09:28.689 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.3.service: Deactivated successfully. 2024-04-05T03:09:28.689 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 systemd[1]: Stopped Ceph osd.3 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:09:28.689 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.3.service: Consumed 30min 21.974s CPU time. 2024-04-05T03:09:28.689 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 systemd[1]: Starting Ceph osd.3 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:09:28.969 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 podman[141511]: 2024-04-05 03:09:28.592771663 +0000 UTC m=+0.018142265 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:28.970 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 podman[141511]: 2024-04-05 03:09:28.727170767 +0000 UTC m=+0.152541368 container create 327fd5d0475fa93630fd88f8b2f519ab54d3d68a202b9c65ee240d15a7e7ddd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_CLEAN=True, RELEASE=squid-0028525, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-05T03:09:28.970 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 podman[141511]: 2024-04-05 03:09:28.970634929 +0000 UTC m=+0.396005531 container init 327fd5d0475fa93630fd88f8b2f519ab54d3d68a202b9c65ee240d15a7e7ddd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0) 2024-04-05T03:09:29.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:28 smithi150 podman[141511]: 2024-04-05 03:09:28.976236171 +0000 UTC m=+0.401606775 container start 327fd5d0475fa93630fd88f8b2f519ab54d3d68a202b9c65ee240d15a7e7ddd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:09:29.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 podman[141511]: 2024-04-05 03:09:29.06673791 +0000 UTC m=+0.492108525 container attach 327fd5d0475fa93630fd88f8b2f519ab54d3d68a202b9c65ee240d15a7e7ddd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403) 2024-04-05T03:09:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:29 smithi196 ceph-mon[91352]: pgmap v481: 73 pgs: 2 active+undersized+degraded, 18 active+undersized, 2 stale+active+clean, 51 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-05T03:09:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:29 smithi196 ceph-mon[91352]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-mon[107701]: pgmap v481: 73 pgs: 2 active+undersized+degraded, 18 active+undersized, 2 stale+active+clean, 51 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-05T03:09:29.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-mon[107701]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:29.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-mon[103445]: pgmap v481: 73 pgs: 2 active+undersized+degraded, 18 active+undersized, 2 stale+active+clean, 51 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3/72 objects degraded (4.167%) 2024-04-05T03:09:29.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-mon[103445]: Health check failed: Degraded data redundancy: 3/72 objects degraded (4.167%), 2 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 bash[141511]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-05T03:09:29.728 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-05T03:09:29.729 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-05T03:09:29.729 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-05T03:09:29.729 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate[141527]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-05T03:09:29.729 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:29 smithi150 podman[141511]: 2024-04-05 03:09:29.441226311 +0000 UTC m=+0.866596917 container died 327fd5d0475fa93630fd88f8b2f519ab54d3d68a202b9c65ee240d15a7e7ddd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, GIT_CLEAN=True, ceph=True) 2024-04-05T03:09:30.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:30 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/324895630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:30.643 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:30 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/324895630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:30.644 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:30 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/324895630' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:30.644 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:30 smithi150 podman[141662]: 2024-04-05 03:09:30.434139063 +0000 UTC m=+0.987662007 container remove 327fd5d0475fa93630fd88f8b2f519ab54d3d68a202b9c65ee240d15a7e7ddd8 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=squid-0028525, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-05T03:09:30.911 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:30 smithi150 podman[141759]: 2024-04-05 03:09:30.547867117 +0000 UTC m=+0.017868793 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:30.911 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:30 smithi150 podman[141759]: 2024-04-05 03:09:30.676779208 +0000 UTC m=+0.146780944 container create b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-05T03:09:30.911 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:30 smithi150 podman[141759]: 2024-04-05 03:09:30.912127155 +0000 UTC m=+0.382128831 container init b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-05T03:09:31.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:30 smithi150 podman[141759]: 2024-04-05 03:09:30.917799331 +0000 UTC m=+0.387801009 container start b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2) 2024-04-05T03:09:31.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:30 smithi150 ceph-osd[141797]: -- 172.21.15.150:0/2101715318 <== mon.2 v2:172.21.15.150:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c801fef680 con 0x55c801fb9400 2024-04-05T03:09:31.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:31 smithi150 bash[141759]: b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 2024-04-05T03:09:31.227 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:31 smithi150 systemd[1]: Started Ceph osd.3 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:09:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:31 smithi196 ceph-mon[91352]: pgmap v482: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:31 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:31 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:31 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:31.727 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:09:31.463+0000 7f77d63697c0 -1 Falling back to public interface 2024-04-05T03:09:31.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:09:31] "GET /metrics HTTP/1.1" 200 34118 "" "Prometheus/2.43.0" 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[103445]: pgmap v482: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[107701]: pgmap v482: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:31.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:31 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:33.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:33 smithi196 ceph-mon[91352]: pgmap v483: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:33.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:33 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1548254957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:33.713 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:33 smithi150 ceph-mon[107701]: pgmap v483: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:33.713 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:33 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1548254957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:33.713 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:33 smithi150 ceph-mon[103445]: pgmap v483: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:33.713 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:33 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1548254957' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:35 smithi196 ceph-mon[91352]: pgmap v484: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3275551208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:35 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[107701]: pgmap v484: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3275551208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[103445]: pgmap v484: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3275551208' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:35.728 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:35 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.237 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:09:37.077+0000 7f77d63697c0 -1 osd.3 135 log_to_monitors true 2024-04-05T03:09:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:37 smithi196 ceph-mon[91352]: pgmap v485: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:37 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:37 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:37 smithi196 ceph-mon[91352]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:09:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:37 smithi196 ceph-mon[91352]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:37.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:37 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/424982171' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:37.684 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[103445]: pgmap v485: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[103445]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[103445]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/424982171' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[107701]: pgmap v485: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[107701]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-05T03:09:37.685 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[107701]: Health check update: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded (PG_DEGRADED) 2024-04-05T03:09:37.686 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/424982171' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:37.686 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:09:37 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:09:37.372+0000 7f77c77c1700 -1 osd.3 135 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-05T03:09:38.362 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[103445]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T03:09:38.362 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[103445]: failed to encode map e138 with expected crc 2024-04-05T03:09:38.362 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[103445]: osdmap e138: 8 total, 7 up, 8 in 2024-04-05T03:09:38.362 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[103445]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:38 smithi196 ceph-mon[91352]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T03:09:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:38 smithi196 ceph-mon[91352]: failed to encode map e138 with expected crc 2024-04-05T03:09:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:38 smithi196 ceph-mon[91352]: osdmap e138: 8 total, 7 up, 8 in 2024-04-05T03:09:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:38 smithi196 ceph-mon[91352]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[107701]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-05T03:09:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[107701]: failed to encode map e138 with expected crc 2024-04-05T03:09:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[107701]: osdmap e138: 8 total, 7 up, 8 in 2024-04-05T03:09:38.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:38 smithi150 ceph-mon[107701]: from='osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi150", "root=default"]}]: dispatch 2024-04-05T03:09:39.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: pgmap v487: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:39.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:09:39.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224] boot 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: osdmap e139: 8 total, 8 up, 8 in 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[103445]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: pgmap v487: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224] boot 2024-04-05T03:09:39.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: osdmap e139: 8 total, 8 up, 8 in 2024-04-05T03:09:39.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:09:39.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-mon[107701]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.571 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 4 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: pgmap v487: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: osd.3 [v2:172.21.15.150:6826/3078047224,v1:172.21.15.150:6827/3078047224] boot 2024-04-05T03:09:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: osdmap e139: 8 total, 8 up, 8 in 2024-04-05T03:09:39.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-05T03:09:39.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: failed to encode map e139 with expected crc 2024-04-05T03:09:39.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:39 smithi196 ceph-mon[91352]: failed to encode map e139 with expected crc 2024-04-05T03:09:40.180 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:39 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:09:39.975+0000 7f45dfa59700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: osdmap e140: 8 total, 8 up, 8 in 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3026591095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:09:40.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: osdmap e140: 8 total, 8 up, 8 in 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3026591095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.479 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.480 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:09:40.519 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch upgrade pause' 2024-04-05T03:09:40.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: osdmap e140: 8 total, 8 up, 8 in 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: failed to encode map e140 with expected crc 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3026591095' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:40.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:40.638 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:09:40.796 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[103445]: pgmap v490: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[103445]: pgmap v491: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[103445]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[103445]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[107701]: pgmap v490: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[107701]: pgmap v491: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[107701]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:09:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[107701]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:41.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:41 smithi196 ceph-mon[91352]: pgmap v490: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:41 smithi196 ceph-mon[91352]: pgmap v491: 73 pgs: 3 active+undersized+degraded, 24 active+undersized, 46 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:41 smithi196 ceph-mon[91352]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-05T03:09:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:41 smithi196 ceph-mon[91352]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-04-05T03:09:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:41 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:41.922 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:09:41] "GET /metrics HTTP/1.1" 200 33964 "" "Prometheus/2.43.0" 2024-04-05T03:09:42.696 INFO:teuthology.orchestra.run.smithi150.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:43.282 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:09:43.547 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:43.810 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[103445]: pgmap v492: 73 pgs: 3 active+undersized+degraded, 18 active+undersized, 52 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:43.810 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[103445]: from='client.45131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:43.810 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[103445]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:43.810 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:43.811 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[107701]: pgmap v492: 73 pgs: 3 active+undersized+degraded, 18 active+undersized, 52 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:43.811 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[107701]: from='client.45131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:43.811 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[107701]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:43.811 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:43 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:43 smithi196 ceph-mon[91352]: pgmap v492: 73 pgs: 3 active+undersized+degraded, 18 active+undersized, 52 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4/72 objects degraded (5.556%) 2024-04-05T03:09:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:43 smithi196 ceph-mon[91352]: from='client.45131 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:43 smithi196 ceph-mon[91352]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:09:44.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:43 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:44.969 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:44 smithi150 ceph-mon[103445]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:44.969 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:44 smithi150 ceph-mon[103445]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-05T03:09:44.969 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:44 smithi150 ceph-mon[107701]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:44.969 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:44 smithi150 ceph-mon[107701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-05T03:09:45.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:44 smithi196 ceph-mon[91352]: pgmap v493: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:45.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:44 smithi196 ceph-mon[91352]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/72 objects degraded (5.556%), 3 pgs degraded) 2024-04-05T03:09:45.415 INFO:teuthology.orchestra.run.smithi150.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:alertmanager.a smithi150 *:9093,9094 running (16m) 10s ago 20m 26.0M - 0.25.0 c8568f914cd2 5c82bec44fb9 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:grafana.a smithi196 *:3000 error 10s ago 20m - - 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi150.jnrwgs smithi150 running (19m) 10s ago 19m 19.5M - 17.2.7-805-g6fda41ff a49113eacc58 cfa13d4de7a1 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi196.eohomm smithi196 running (19m) 10s ago 19m 19.2M - 17.2.7-805-g6fda41ff a49113eacc58 e3ce4d99c3da 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.x smithi196 *:8443,9283,8765 running (14m) 10s ago 24m 456M - 19.0.0-2013-g0028525c d8c35eaf3c25 7ca2051bb392 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.y smithi150 *:8443,9283,8765 running (15m) 10s ago 26m 515M - 19.0.0-2013-g0028525c d8c35eaf3c25 0d5b82cc2eb2 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mon.a smithi150 running (14m) 10s ago 26m 67.6M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 39df812ce046 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mon.b smithi196 running (10m) 10s ago 25m 51.3M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 51bbb41f1c91 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:mon.c smithi150 running (12m) 10s ago 25m 53.8M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 74316b4d98ae 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.a smithi150 *:9100 running (16m) 10s ago 20m 16.7M - 1.5.0 0da6a335fe13 dc5e73ff394f 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.b smithi196 *:9100 running (15m) 10s ago 20m 16.7M - 1.5.0 0da6a335fe13 0d264938c335 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:osd.0 smithi150 running (8m) 10s ago 24m 672M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 415d151c052d 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:osd.1 smithi150 running (5m) 10s ago 24m 261M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 033996a9bd59 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:osd.2 smithi150 running (50s) 10s ago 23m 74.8M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 155b19d15ab1 2024-04-05T03:09:45.416 INFO:teuthology.orchestra.run.smithi150.stdout:osd.3 smithi150 running (14s) 10s ago 23m 14.9M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 b1a0d7c6725e 2024-04-05T03:09:45.417 INFO:teuthology.orchestra.run.smithi150.stdout:osd.4 smithi196 running (23m) 10s ago 23m 540M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b21803209415 2024-04-05T03:09:45.417 INFO:teuthology.orchestra.run.smithi150.stdout:osd.5 smithi196 running (22m) 10s ago 22m 938M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5415e890a870 2024-04-05T03:09:45.417 INFO:teuthology.orchestra.run.smithi150.stdout:osd.6 smithi196 running (22m) 10s ago 22m 671M 2480M 17.2.7-805-g6fda41ff a49113eacc58 85a4b25b62cf 2024-04-05T03:09:45.417 INFO:teuthology.orchestra.run.smithi150.stdout:osd.7 smithi196 running (21m) 10s ago 21m 580M 2480M 17.2.7-805-g6fda41ff a49113eacc58 ed60a8dad4f7 2024-04-05T03:09:45.417 INFO:teuthology.orchestra.run.smithi150.stdout:prometheus.a smithi196 *:9095 running (14m) 10s ago 21m 64.5M - 2.43.0 a07b618ecd1d dc41e7016bef 2024-04-05T03:09:46.062 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:45 smithi150 ceph-mon[107701]: from='client.45137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:46.063 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:45 smithi150 ceph-mon[103445]: from='client.45137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:46.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:45 smithi196 ceph-mon[91352]: from='client.45137 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:46.367 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph orch ps' 2024-04-05T03:09:46.629 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:47.023 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:46 smithi150 ceph-mon[103445]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:47.024 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:46 smithi150 ceph-mon[107701]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:47.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:46 smithi196 ceph-mon[91352]: pgmap v494: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:alertmanager.a smithi150 *:9093,9094 running (16m) 13s ago 20m 26.0M - 0.25.0 c8568f914cd2 5c82bec44fb9 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:grafana.a smithi196 *:3000 error 13s ago 20m - - 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi150.jnrwgs smithi150 running (19m) 13s ago 19m 19.5M - 17.2.7-805-g6fda41ff a49113eacc58 cfa13d4de7a1 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:mds.foo.smithi196.eohomm smithi196 running (19m) 13s ago 19m 19.2M - 17.2.7-805-g6fda41ff a49113eacc58 e3ce4d99c3da 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.x smithi196 *:8443,9283,8765 running (14m) 13s ago 25m 456M - 19.0.0-2013-g0028525c d8c35eaf3c25 7ca2051bb392 2024-04-05T03:09:48.582 INFO:teuthology.orchestra.run.smithi150.stdout:mgr.y smithi150 *:8443,9283,8765 running (15m) 13s ago 26m 515M - 19.0.0-2013-g0028525c d8c35eaf3c25 0d5b82cc2eb2 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:mon.a smithi150 running (14m) 13s ago 26m 67.6M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 39df812ce046 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:mon.b smithi196 running (10m) 13s ago 25m 51.3M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 51bbb41f1c91 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:mon.c smithi150 running (12m) 13s ago 25m 53.8M 2048M 19.0.0-2013-g0028525c d8c35eaf3c25 74316b4d98ae 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.a smithi150 *:9100 running (16m) 13s ago 20m 16.7M - 1.5.0 0da6a335fe13 dc5e73ff394f 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:node-exporter.b smithi196 *:9100 running (15m) 13s ago 20m 16.7M - 1.5.0 0da6a335fe13 0d264938c335 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.0 smithi150 running (8m) 13s ago 24m 672M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 415d151c052d 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.1 smithi150 running (5m) 13s ago 24m 261M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 033996a9bd59 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.2 smithi150 running (53s) 13s ago 23m 74.8M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 155b19d15ab1 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.3 smithi150 running (18s) 13s ago 23m 14.9M 2480M 19.0.0-2013-g0028525c d8c35eaf3c25 b1a0d7c6725e 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.4 smithi196 running (23m) 13s ago 23m 540M 2480M 17.2.7-805-g6fda41ff a49113eacc58 b21803209415 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.5 smithi196 running (22m) 13s ago 22m 938M 2480M 17.2.7-805-g6fda41ff a49113eacc58 5415e890a870 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.6 smithi196 running (22m) 13s ago 22m 671M 2480M 17.2.7-805-g6fda41ff a49113eacc58 85a4b25b62cf 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:osd.7 smithi196 running (21m) 13s ago 21m 580M 2480M 17.2.7-805-g6fda41ff a49113eacc58 ed60a8dad4f7 2024-04-05T03:09:48.583 INFO:teuthology.orchestra.run.smithi150.stdout:prometheus.a smithi196 *:9095 running (14m) 13s ago 21m 64.5M - 2.43.0 a07b618ecd1d dc41e7016bef 2024-04-05T03:09:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:49 smithi150 ceph-mon[103445]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:49.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:49 smithi150 ceph-mon[103445]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:49.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:49 smithi150 ceph-mon[107701]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:49.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:49 smithi150 ceph-mon[107701]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:49.278 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -e sha1=0028525c15e19309c20289c609e4fb9d0b12dd06 -- bash -c 'ceph versions' 2024-04-05T03:09:49.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:49 smithi196 ceph-mon[91352]: pgmap v495: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:49.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:49 smithi196 ceph-mon[91352]: from='client.45143 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-05T03:09:49.535 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:51.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:51 smithi150 ceph-mon[103445]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:51.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:51 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:51.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:51 smithi150 ceph-mon[107701]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:51.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:51 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:51.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:51 smithi196 ceph-mon[91352]: pgmap v496: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:51.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:51 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout:{ 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "mon": { 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 3 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "mgr": { 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 2 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "osd": { 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)": 4, 2024-04-05T03:09:51.627 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 4 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "mds": { 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)": 2 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: }, 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "overall": { 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 17.2.7-805-g6fda41ff (6fda41ff74b7f038ecae6f40cc5fca107a3d4207) quincy (stable)": 6, 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: "ceph version 19.0.0-2013-g0028525c (0028525c15e19309c20289c609e4fb9d0b12dd06) squid (dev)": 9 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout: } 2024-04-05T03:09:51.628 INFO:teuthology.orchestra.run.smithi150.stdout:} 2024-04-05T03:09:51.959 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:09:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:09:51] "GET /metrics HTTP/1.1" 200 33964 "" "Prometheus/2.43.0" 2024-04-05T03:09:52.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1201591289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:52.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1201591289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:52.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1201591289' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-05T03:09:52.414 DEBUG:teuthology.parallel:result is None 2024-04-05T03:09:52.414 INFO:teuthology.run_tasks:Running task print... 2024-04-05T03:09:52.425 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-05T03:09:52.425 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-05T03:09:52.491 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-05T03:09:52.491 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-05T03:09:52.492 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-05T03:09:52.765 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:53.188 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:53 smithi150 ceph-mon[103445]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:53.188 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:53 smithi150 ceph-mon[107701]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:53.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:53 smithi196 ceph-mon[91352]: pgmap v497: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:54.692 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:09:55.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:55.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3639507379' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3639507379' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:55.292 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:55.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:55.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:55.293 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:55 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:55.314 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":40,"flags":0,"active_gid":25295,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6800","nonce":3856626881},{"type":"v1","addr":"172.21.15.150:6801","nonce":3856626881}]},"active_addr":"172.21.15.150:6801/3856626881","active_change":"2024-04-05T02:55:55.494326+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25210,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.150:8443/","prometheus":"http://172.21.15.150:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":83,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2550749382}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2529571168}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":3980919664}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.150:0","nonce":2224002175}]}]} 2024-04-05T03:09:55.319 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-05T03:09:55.319 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-05T03:09:55.319 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:09:55.576 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: pgmap v498: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3639507379' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:09:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:09:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:09:55.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:55 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[103445]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[103445]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[103445]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[103445]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:56.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[107701]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[107701]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[107701]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[107701]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:56.228 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:56 smithi196 ceph-mon[91352]: pgmap v499: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:56 smithi196 ceph-mon[91352]: pgmap v500: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:56 smithi196 ceph-mon[91352]: pgmap v501: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:56 smithi196 ceph-mon[91352]: pgmap v502: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:56 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:09:56.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:09:57.589 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:09:57.589 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":140,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T03:09:39.358066+0000","last_up_change":"2024-04-05T03:09:38.350878+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T02:48:34.437290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T02:49:46.120483+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T02:49:46.414509+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":94,"up_thru":139,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6803","nonce":4017397590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6805","nonce":4017397590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6809","nonce":4017397590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6807","nonce":4017397590}]},"public_addr":"172.21.15.150:6803/4017397590","cluster_addr":"172.21.15.150:6805/4017397590","heartbeat_back_addr":"172.21.15.150:6809/4017397590","heartbeat_front_addr":"172.21.15.150:6807/4017397590","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":115,"up_from":119,"up_thru":139,"down_at":116,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6811","nonce":3505931467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6813","nonce":3505931467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6817","nonce":3505931467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6815","nonce":3505931467}]},"public_addr":"172.21.15.150:6811/3505931467","cluster_addr":"172.21.15.150:6813/3505931467","heartbeat_back_addr":"172.21.15.150:6817/3505931467","heartbeat_front_addr":"172.21.15.150:6815/3505931467","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":130,"up_from":134,"up_thru":139,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6819","nonce":1032167350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6821","nonce":1032167350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6825","nonce":1032167350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6823","nonce":1032167350}]},"public_addr":"172.21.15.150:6819/1032167350","cluster_addr":"172.21.15.150:6821/1032167350","heartbeat_back_addr":"172.21.15.150:6825/1032167350","heartbeat_front_addr":"172.21.15.150:6823/1032167350","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":135,"up_from":139,"up_thru":139,"down_at":136,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6827","nonce":3078047224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6829","nonce":3078047224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6833","nonce":3078047224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6831","nonce":3078047224}]},"public_addr":"172.21.15.150:6827/3078047224","cluster_addr":"172.21.15.150:6829/3078047224","heartbeat_back_addr":"172.21.15.150:6833/3078047224","heartbeat_front_addr":"172.21.15.150:6831/3078047224","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":136,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":115},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":130},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":135},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.0","mappings":[{"from":3,"to":5}]},{"pgid":"4.1","mappings":[{"from":4,"to":0}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.c","mappings":[{"from":3,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":4,"to":0}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5},{"from":3,"to":0}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:0/2745855992":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/3549400810":"2024-04-06T02:48:32.814681+0000","172.21.15.196:0/2114711778":"2024-04-06T02:54:50.772230+0000","172.21.15.150:0/294204367":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2202195129":"2024-04-06T02:48:32.814681+0000","172.21.15.196:0/3795737016":"2024-04-06T02:54:50.772230+0000","172.21.15.150:6801/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/2441406623":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/3079783839":"2024-04-06T02:55:55.494032+0000","172.21.15.196:0/3923077527":"2024-04-06T02:54:50.772230+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/11242322":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/776248876":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/3187799515":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/3029771974":"2024-04-06T02:55:55.494032+0000","172.21.15.150:6800/1834837291":"2024-04-06T02:52:49.942247+0000","172.21.15.150:6801/1834837291":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/2603883274":"2024-04-06T02:52:49.942247+0000","172.21.15.150:6801/3079783839":"2024-04-06T02:55:55.494032+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/222441202":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/198010180":"2024-04-06T02:52:49.942247+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/3200004174":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/4085167408":"2024-04-06T02:55:55.494032+0000","172.21.15.196:6834/1747214673":"2024-04-06T02:54:50.772230+0000","172.21.15.196:6835/1747214673":"2024-04-06T02:54:50.772230+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.196:0/3398877659":"2024-04-06T02:54:50.772230+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:09:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:58 smithi150 ceph-mon[103445]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:09:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/177089939' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:09:58.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:58 smithi150 ceph-mon[107701]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:09:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/177089939' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:09:58.267 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-05T03:09:58.267 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-05T03:09:58.526 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:09:58.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:58 smithi196 ceph-mon[91352]: pgmap v503: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:09:58.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:09:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/177089939' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:10:00.304 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:10:00.305 INFO:teuthology.orchestra.run.smithi150.stdout:{"epoch":140,"fsid":"15e4009a-f2f6-11ee-b649-cb9ed24678a4","created":"2024-04-05T02:42:50.693464+0000","modified":"2024-04-05T03:09:39.358066+0000","last_up_change":"2024-04-05T03:09:38.350878+0000","last_in_change":"2024-04-05T02:47:43.242098+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":8,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-05T02:45:59.176818+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-05T02:48:34.437290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-05T02:49:46.120483+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-05T02:49:46.414509+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.75,"score_stable":1.75,"optimal_score":1,"raw_score_acting":1.75,"raw_score_stable":1.75,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"97c9b0a6-8cf6-4310-87a7-cf7c0eb848ff","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":88,"up_from":94,"up_thru":139,"down_at":89,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6802","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6803","nonce":4017397590}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6804","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6805","nonce":4017397590}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6808","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6809","nonce":4017397590}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6806","nonce":4017397590},{"type":"v1","addr":"172.21.15.150:6807","nonce":4017397590}]},"public_addr":"172.21.15.150:6803/4017397590","cluster_addr":"172.21.15.150:6805/4017397590","heartbeat_back_addr":"172.21.15.150:6809/4017397590","heartbeat_front_addr":"172.21.15.150:6807/4017397590","state":["exists","up"]},{"osd":1,"uuid":"f1257fa7-7dac-4fed-8553-40a62778f3ec","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":115,"up_from":119,"up_thru":139,"down_at":116,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6810","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6811","nonce":3505931467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6812","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6813","nonce":3505931467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6816","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6817","nonce":3505931467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6814","nonce":3505931467},{"type":"v1","addr":"172.21.15.150:6815","nonce":3505931467}]},"public_addr":"172.21.15.150:6811/3505931467","cluster_addr":"172.21.15.150:6813/3505931467","heartbeat_back_addr":"172.21.15.150:6817/3505931467","heartbeat_front_addr":"172.21.15.150:6815/3505931467","state":["exists","up"]},{"osd":2,"uuid":"9c672159-ed53-4d76-acd4-f29bbdb1483f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":130,"up_from":134,"up_thru":139,"down_at":131,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6818","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6819","nonce":1032167350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6820","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6821","nonce":1032167350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6824","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6825","nonce":1032167350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6822","nonce":1032167350},{"type":"v1","addr":"172.21.15.150:6823","nonce":1032167350}]},"public_addr":"172.21.15.150:6819/1032167350","cluster_addr":"172.21.15.150:6821/1032167350","heartbeat_back_addr":"172.21.15.150:6825/1032167350","heartbeat_front_addr":"172.21.15.150:6823/1032167350","state":["exists","up"]},{"osd":3,"uuid":"bac5c8a2-380b-4767-8305-79febfebf621","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":135,"up_from":139,"up_thru":139,"down_at":136,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6826","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6827","nonce":3078047224}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6828","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6829","nonce":3078047224}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6832","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6833","nonce":3078047224}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.150:6830","nonce":3078047224},{"type":"v1","addr":"172.21.15.150:6831","nonce":3078047224}]},"public_addr":"172.21.15.150:6827/3078047224","cluster_addr":"172.21.15.150:6829/3078047224","heartbeat_back_addr":"172.21.15.150:6833/3078047224","heartbeat_front_addr":"172.21.15.150:6831/3078047224","state":["exists","up"]},{"osd":4,"uuid":"361bfd73-2a77-4096-be7e-37c137d35142","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6800","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6801","nonce":929114274}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6802","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6803","nonce":929114274}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6806","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6807","nonce":929114274}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6804","nonce":929114274},{"type":"v1","addr":"172.21.15.196:6805","nonce":929114274}]},"public_addr":"172.21.15.196:6801/929114274","cluster_addr":"172.21.15.196:6803/929114274","heartbeat_back_addr":"172.21.15.196:6807/929114274","heartbeat_front_addr":"172.21.15.196:6805/929114274","state":["exists","up"]},{"osd":5,"uuid":"845873d4-e80b-4f74-b331-5f09131f995f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6808","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6809","nonce":2903834894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6810","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6811","nonce":2903834894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6814","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6815","nonce":2903834894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6812","nonce":2903834894},{"type":"v1","addr":"172.21.15.196:6813","nonce":2903834894}]},"public_addr":"172.21.15.196:6809/2903834894","cluster_addr":"172.21.15.196:6811/2903834894","heartbeat_back_addr":"172.21.15.196:6815/2903834894","heartbeat_front_addr":"172.21.15.196:6813/2903834894","state":["exists","up"]},{"osd":6,"uuid":"f15bc1e1-770a-47eb-8ef6-5dc6bec7c3dc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":136,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6816","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6817","nonce":471833024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6818","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6819","nonce":471833024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6822","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6823","nonce":471833024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6820","nonce":471833024},{"type":"v1","addr":"172.21.15.196:6821","nonce":471833024}]},"public_addr":"172.21.15.196:6817/471833024","cluster_addr":"172.21.15.196:6819/471833024","heartbeat_back_addr":"172.21.15.196:6823/471833024","heartbeat_front_addr":"172.21.15.196:6821/471833024","state":["exists","up"]},{"osd":7,"uuid":"5267c69d-e13b-45b4-a15e-91a620aa0647","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":139,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6824","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6825","nonce":3556112114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6826","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6827","nonce":3556112114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6830","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6831","nonce":3556112114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.196:6828","nonce":3556112114},{"type":"v1","addr":"172.21.15.196:6829","nonce":3556112114}]},"public_addr":"172.21.15.196:6825/3556112114","cluster_addr":"172.21.15.196:6827/3556112114","heartbeat_back_addr":"172.21.15.196:6831/3556112114","heartbeat_front_addr":"172.21.15.196:6829/3556112114","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:09.165128+0000","dead_epoch":88},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:32.311996+0000","dead_epoch":115},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:45:56.082162+0000","dead_epoch":130},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:20.962875+0000","dead_epoch":135},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:46:43.712336+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:08.204431+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:33.051250+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-05T02:47:58.896809+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.0","mappings":[{"from":3,"to":5}]},{"pgid":"4.1","mappings":[{"from":4,"to":0}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.c","mappings":[{"from":3,"to":5}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.13","mappings":[{"from":4,"to":0}]},{"pgid":"4.14","mappings":[{"from":1,"to":2}]},{"pgid":"4.1b","mappings":[{"from":3,"to":5}]},{"pgid":"4.1c","mappings":[{"from":1,"to":5},{"from":3,"to":0}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.150:0/294204367":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/776248876":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/2745855992":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/3438297848":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/3549400810":"2024-04-06T02:48:32.814681+0000","172.21.15.196:0/2114711778":"2024-04-06T02:54:50.772230+0000","172.21.15.150:0/3102218940":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/2202195129":"2024-04-06T02:48:32.814681+0000","172.21.15.196:0/3795737016":"2024-04-06T02:54:50.772230+0000","172.21.15.150:6801/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:6800/3079783839":"2024-04-06T02:55:55.494032+0000","172.21.15.150:6800/3263117728":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/2441406623":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/3394016813":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/11242322":"2024-04-06T02:52:49.942247+0000","172.21.15.150:6801/1834837291":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/222441202":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/198010180":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/1494863409":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6801/3079783839":"2024-04-06T02:55:55.494032+0000","172.21.15.150:6801/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:0/2603883274":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/3187799515":"2024-04-06T02:55:55.494032+0000","172.21.15.150:0/3029771974":"2024-04-06T02:55:55.494032+0000","172.21.15.150:6800/1834837291":"2024-04-06T02:52:49.942247+0000","172.21.15.150:0/3200004174":"2024-04-06T02:48:32.814681+0000","172.21.15.150:0/4085167408":"2024-04-06T02:55:55.494032+0000","172.21.15.196:6834/1747214673":"2024-04-06T02:54:50.772230+0000","172.21.15.196:6835/1747214673":"2024-04-06T02:54:50.772230+0000","172.21.15.150:0/2891804917":"2024-04-06T02:43:22.333440+0000","172.21.15.150:0/576292192":"2024-04-06T02:43:46.857858+0000","172.21.15.196:0/3923077527":"2024-04-06T02:54:50.772230+0000","172.21.15.196:0/3398877659":"2024-04-06T02:54:50.772230+0000","172.21.15.150:6800/2688470902":"2024-04-06T02:43:46.857858+0000","172.21.15.150:6800/1491399316":"2024-04-06T02:43:22.333440+0000","172.21.15.150:6801/1491399316":"2024-04-06T02:43:22.333440+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-05T03:10:00.434 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:00 smithi150 ceph-mon[103445]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:00.434 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:00 smithi150 ceph-mon[103445]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:10:00.434 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:00 smithi150 ceph-mon[107701]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:00.434 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:00 smithi150 ceph-mon[107701]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:10:00.438 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:00 smithi196 ceph-mon[91352]: pgmap v504: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:00.438 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:00 smithi196 ceph-mon[91352]: overall HEALTH_WARN 1 failed cephadm daemon(s) 2024-04-05T03:10:00.930 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-05T03:10:00.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-05T03:10:00.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-05T03:10:00.931 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-05T03:10:00.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-05T03:10:00.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-05T03:10:00.932 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.6 flush_pg_stats 2024-04-05T03:10:00.933 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph tell osd.7 flush_pg_stats 2024-04-05T03:10:01.154 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:01 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/947639145' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:10:01.419 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.421 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.476 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:01 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/947639145' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:10:01.477 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.478 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.480 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.481 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:01.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:01 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/947639145' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-05T03:10:01.883 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:10:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:10:01] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-04-05T03:10:02.430 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:02 smithi150 ceph-mon[103445]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:02.430 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:02 smithi150 ceph-mon[107701]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:02.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:02 smithi196 ceph-mon[91352]: pgmap v505: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:04.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:04 smithi150 ceph-mon[103445]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:04.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:04 smithi150 ceph-mon[107701]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:04.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:04 smithi196 ceph-mon[91352]: pgmap v506: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:05.408 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:06.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:06 smithi150 ceph-mon[103445]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:06.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:06 smithi150 ceph-mon[107701]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:06.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:06 smithi196 ceph-mon[91352]: pgmap v507: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:08.387 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:08 smithi150 ceph-mon[107701]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:08.387 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:08 smithi150 ceph-mon[103445]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:08.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:08 smithi196 ceph-mon[91352]: pgmap v508: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:10.365 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:10 smithi150 ceph-mon[107701]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:10.366 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:10 smithi150 ceph-mon[103445]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:10.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:10 smithi196 ceph-mon[91352]: pgmap v509: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:11.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:11.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:11.825 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:10:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:10:11] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-04-05T03:10:12.351 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:12 smithi150 ceph-mon[107701]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.351 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:12 smithi150 ceph-mon[103445]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:12.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:12 smithi196 ceph-mon[91352]: pgmap v510: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:13.243 INFO:teuthology.orchestra.run.smithi150.stdout:167503724821 2024-04-05T03:10:13.243 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-05T03:10:14.440 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:14 smithi150 ceph-mon[103445]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:14.441 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:14 smithi150 ceph-mon[107701]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:14.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:14 smithi196 ceph-mon[91352]: pgmap v511: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:14.862 INFO:teuthology.orchestra.run.smithi150.stdout:201863463187 2024-04-05T03:10:14.863 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.6 2024-04-05T03:10:15.231 INFO:teuthology.orchestra.run.smithi150.stdout:511101108291 2024-04-05T03:10:15.231 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-05T03:10:16.413 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:16 smithi150 ceph-mon[107701]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:16.413 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:16 smithi150 ceph-mon[103445]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:16.438 INFO:teuthology.orchestra.run.smithi150.stdout:575525617679 2024-04-05T03:10:16.438 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-05T03:10:16.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:16 smithi196 ceph-mon[91352]: pgmap v512: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:17.183 INFO:teuthology.orchestra.run.smithi150.stdout:403726925934 2024-04-05T03:10:17.184 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-05T03:10:17.908 INFO:teuthology.orchestra.run.smithi150.stdout:597000454153 2024-04-05T03:10:17.909 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-05T03:10:18.188 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:18 smithi150 ceph-mon[107701]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:18.320 INFO:teuthology.orchestra.run.smithi150.stdout:236223201549 2024-04-05T03:10:18.321 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.7 2024-04-05T03:10:18.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:18 smithi150 ceph-mon[103445]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:18.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:18 smithi196 ceph-mon[91352]: pgmap v513: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:18.838 INFO:teuthology.orchestra.run.smithi150.stdout:133143986460 2024-04-05T03:10:18.838 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-05T03:10:18.883 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:18.883 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:18.884 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:18.886 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:20.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:20 smithi150 ceph-mon[103445]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:20.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:20 smithi150 ceph-mon[107701]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:20.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:20 smithi196 ceph-mon[91352]: pgmap v514: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:21.845 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:21.941 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:10:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:10:21] "GET /metrics HTTP/1.1" 200 34116 "" "Prometheus/2.43.0" 2024-04-05T03:10:22.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:22 smithi150 ceph-mon[103445]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:22.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:22 smithi150 ceph-mon[107701]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:22 smithi196 ceph-mon[91352]: pgmap v515: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:24.006 INFO:teuthology.orchestra.run.smithi150.stdout:201863463189 2024-04-05T03:10:24.404 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:24.404 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:24.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:24 smithi150 ceph-mon[103445]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:24.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:24 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2075506257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:10:24.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:24 smithi150 ceph-mon[107701]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:24.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:24 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2075506257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:10:24.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:24 smithi196 ceph-mon[91352]: pgmap v516: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:24.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:24 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2075506257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-05T03:10:25.066 INFO:teuthology.orchestra.run.smithi150.stdout:575525617682 2024-04-05T03:10:25.240 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463187 got 201863463189 for osd.6 2024-04-05T03:10:25.240 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:25.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:25 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3586786766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:10:25.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:25 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3586786766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:10:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:25 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3586786766' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-05T03:10:26.348 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:26 smithi150 ceph-mon[103445]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:26.349 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:26.349 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:26 smithi150 ceph-mon[107701]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:26.349 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:26.372 INFO:tasks.cephadm.ceph_manager.ceph:need seq 575525617679 got 575525617682 for osd.2 2024-04-05T03:10:26.372 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:26.376 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:26 smithi196 ceph-mon[91352]: pgmap v517: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:28.003 INFO:teuthology.orchestra.run.smithi150.stdout:167503724825 2024-04-05T03:10:28.417 INFO:teuthology.orchestra.run.smithi150.stdout:511101108294 2024-04-05T03:10:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:28 smithi150 ceph-mon[103445]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:28 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1309395960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:10:28.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:28 smithi150 ceph-mon[107701]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:28.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:28 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1309395960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:10:28.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:28 smithi196 ceph-mon[91352]: pgmap v518: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:28.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:28 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1309395960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-05T03:10:29.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/339463201' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:10:29.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/339463201' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:10:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/339463201' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-05T03:10:29.666 INFO:teuthology.orchestra.run.smithi150.stdout:133143986463 2024-04-05T03:10:29.949 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724821 got 167503724825 for osd.5 2024-04-05T03:10:29.949 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:30.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:30 smithi150 ceph-mon[107701]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:30.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:30 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1006588771' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:10:30.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:30 smithi150 ceph-mon[103445]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:30.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:30 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1006588771' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:10:30.537 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:30 smithi196 ceph-mon[91352]: pgmap v519: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:30.537 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:30 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1006588771' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-05T03:10:31.627 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:10:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:10:31] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-04-05T03:10:31.653 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986460 got 133143986463 for osd.4 2024-04-05T03:10:31.653 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:32.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:32 smithi150 ceph-mon[103445]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:32.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:32 smithi150 ceph-mon[107701]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:32.408 INFO:tasks.cephadm.ceph_manager.ceph:need seq 511101108291 got 511101108294 for osd.1 2024-04-05T03:10:32.408 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:32.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:32 smithi196 ceph-mon[91352]: pgmap v520: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:33.206 INFO:teuthology.orchestra.run.smithi150.stdout:597000454156 2024-04-05T03:10:33.296 INFO:teuthology.orchestra.run.smithi150.stdout:403726925937 2024-04-05T03:10:33.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:33 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4070586402' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:10:33.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:33 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4070586402' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:10:33.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:33 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4070586402' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-05T03:10:33.988 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925934 got 403726925937 for osd.0 2024-04-05T03:10:33.988 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:34 smithi150 ceph-mon[103445]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:34.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:34 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1474248407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:10:34.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:34 smithi150 ceph-mon[107701]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:34.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:34 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1474248407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:10:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:34 smithi196 ceph-mon[91352]: pgmap v521: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:34.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:34 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1474248407' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-05T03:10:34.726 INFO:tasks.cephadm.ceph_manager.ceph:need seq 597000454153 got 597000454156 for osd.3 2024-04-05T03:10:34.726 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:35.341 INFO:teuthology.orchestra.run.smithi150.stdout:236223201553 2024-04-05T03:10:35.990 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201549 got 236223201553 for osd.7 2024-04-05T03:10:35.990 DEBUG:teuthology.parallel:result is None 2024-04-05T03:10:35.991 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-05T03:10:35.991 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:10:36.253 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:36.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:36 smithi150 ceph-mon[103445]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:36.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:36 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2227639282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:10:36.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:36 smithi150 ceph-mon[107701]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:36.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:36 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2227639282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:10:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:36 smithi196 ceph-mon[91352]: pgmap v522: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:36 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2227639282' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-05T03:10:38.138 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:10:38.139 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-04-05T03:10:38.556 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:38 smithi150 ceph-mon[107701]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:38.556 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:38 smithi150 ceph-mon[103445]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:38.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:38 smithi196 ceph-mon[91352]: pgmap v523: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:38.890 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":523,"stamp":"2024-04-05T03:10:37.117383+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":301,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":217,"ondisk_log_size":217,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2407124,"kb_used_data":13900,"kb_used_omap":29,"kb_used_meta":2393058,"kb_avail":747554092,"statfs":{"total":767960285184,"available":765495390208,"internally_reserved":0,"allocated":14233600,"data_stored":8872299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30042,"internal_metadata":2450492070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002514"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376505+0000","last_change":"2024-04-05T03:09:04.085323+0000","last_active":"2024-04-05T03:09:38.376505+0000","last_peered":"2024-04-05T03:09:38.376505+0000","last_clean":"2024-04-05T03:09:38.376505+0000","last_became_active":"2024-04-05T03:09:04.084948+0000","last_became_peered":"2024-04-05T03:09:04.084948+0000","last_unstale":"2024-04-05T03:09:38.376505+0000","last_undegraded":"2024-04-05T03:09:38.376505+0000","last_fullsized":"2024-04-05T03:09:38.376505+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:20:46.064821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373362+0000","last_change":"2024-04-05T03:09:39.373362+0000","last_active":"2024-04-05T03:09:39.373362+0000","last_peered":"2024-04-05T03:09:39.373362+0000","last_clean":"2024-04-05T03:09:39.373362+0000","last_became_active":"2024-04-05T03:09:39.372920+0000","last_became_peered":"2024-04-05T03:09:39.372920+0000","last_unstale":"2024-04-05T03:09:39.373362+0000","last_undegraded":"2024-04-05T03:09:39.373362+0000","last_fullsized":"2024-04-05T03:09:39.373362+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:42:38.110269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.379935+0000","last_change":"2024-04-05T03:09:39.379935+0000","last_active":"2024-04-05T03:09:39.379935+0000","last_peered":"2024-04-05T03:09:39.379935+0000","last_clean":"2024-04-05T03:09:39.379935+0000","last_became_active":"2024-04-05T03:09:39.379596+0000","last_became_peered":"2024-04-05T03:09:39.379596+0000","last_unstale":"2024-04-05T03:09:39.379935+0000","last_undegraded":"2024-04-05T03:09:39.379935+0000","last_fullsized":"2024-04-05T03:09:39.379935+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:09:51.368120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":126,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.383940+0000","last_change":"2024-04-05T03:09:39.383940+0000","last_active":"2024-04-05T03:09:39.383940+0000","last_peered":"2024-04-05T03:09:39.383940+0000","last_clean":"2024-04-05T03:09:39.383940+0000","last_became_active":"2024-04-05T03:09:39.383631+0000","last_became_peered":"2024-04-05T03:09:39.383631+0000","last_unstale":"2024-04-05T03:09:39.383940+0000","last_undegraded":"2024-04-05T03:09:39.383940+0000","last_fullsized":"2024-04-05T03:09:39.383940+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:21:33.901632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.374442+0000","last_change":"2024-04-05T03:09:39.374442+0000","last_active":"2024-04-05T03:09:39.374442+0000","last_peered":"2024-04-05T03:09:39.374442+0000","last_clean":"2024-04-05T03:09:39.374442+0000","last_became_active":"2024-04-05T03:09:39.374060+0000","last_became_peered":"2024-04-05T03:09:39.374060+0000","last_unstale":"2024-04-05T03:09:39.374442+0000","last_undegraded":"2024-04-05T03:09:39.374442+0000","last_fullsized":"2024-04-05T03:09:39.374442+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:02:15.911109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378190+0000","last_change":"2024-04-05T03:04:44.547518+0000","last_active":"2024-04-05T03:09:38.378190+0000","last_peered":"2024-04-05T03:09:38.378190+0000","last_clean":"2024-04-05T03:09:38.378190+0000","last_became_active":"2024-04-05T03:04:44.547154+0000","last_became_peered":"2024-04-05T03:04:44.547154+0000","last_unstale":"2024-04-05T03:09:38.378190+0000","last_undegraded":"2024-04-05T03:09:38.378190+0000","last_fullsized":"2024-04-05T03:09:38.378190+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:25:14.595296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378073+0000","last_change":"2024-04-05T03:09:39.378073+0000","last_active":"2024-04-05T03:09:39.378073+0000","last_peered":"2024-04-05T03:09:39.378073+0000","last_clean":"2024-04-05T03:09:39.378073+0000","last_became_active":"2024-04-05T03:09:39.377691+0000","last_became_peered":"2024-04-05T03:09:39.377691+0000","last_unstale":"2024-04-05T03:09:39.378073+0000","last_undegraded":"2024-04-05T03:09:39.378073+0000","last_fullsized":"2024-04-05T03:09:39.378073+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:00:01.207793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373884+0000","last_change":"2024-04-05T03:09:39.373884+0000","last_active":"2024-04-05T03:09:39.373884+0000","last_peered":"2024-04-05T03:09:39.373884+0000","last_clean":"2024-04-05T03:09:39.373884+0000","last_became_active":"2024-04-05T03:09:39.373578+0000","last_became_peered":"2024-04-05T03:09:39.373578+0000","last_unstale":"2024-04-05T03:09:39.373884+0000","last_undegraded":"2024-04-05T03:09:39.373884+0000","last_fullsized":"2024-04-05T03:09:39.373884+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:01:50.207471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371704+0000","last_change":"2024-04-05T03:04:44.542000+0000","last_active":"2024-04-05T03:09:38.371704+0000","last_peered":"2024-04-05T03:09:38.371704+0000","last_clean":"2024-04-05T03:09:38.371704+0000","last_became_active":"2024-04-05T03:04:44.541683+0000","last_became_peered":"2024-04-05T03:04:44.541683+0000","last_unstale":"2024-04-05T03:09:38.371704+0000","last_undegraded":"2024-04-05T03:09:38.371704+0000","last_fullsized":"2024-04-05T03:09:38.371704+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:42:28.072974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.372268+0000","last_change":"2024-04-05T03:09:39.372268+0000","last_active":"2024-04-05T03:09:39.372268+0000","last_peered":"2024-04-05T03:09:39.372268+0000","last_clean":"2024-04-05T03:09:39.372268+0000","last_became_active":"2024-04-05T03:09:39.371920+0000","last_became_peered":"2024-04-05T03:09:39.371920+0000","last_unstale":"2024-04-05T03:09:39.372268+0000","last_undegraded":"2024-04-05T03:09:39.372268+0000","last_fullsized":"2024-04-05T03:09:39.372268+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:31:28.817062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":102,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.377767+0000","last_change":"2024-04-05T03:09:39.377767+0000","last_active":"2024-04-05T03:09:39.377767+0000","last_peered":"2024-04-05T03:09:39.377767+0000","last_clean":"2024-04-05T03:09:39.377767+0000","last_became_active":"2024-04-05T03:09:39.377354+0000","last_became_peered":"2024-04-05T03:09:39.377354+0000","last_unstale":"2024-04-05T03:09:39.377767+0000","last_undegraded":"2024-04-05T03:09:39.377767+0000","last_fullsized":"2024-04-05T03:09:39.377767+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:27:22.024303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.365131+0000","last_change":"2024-04-05T03:09:04.077448+0000","last_active":"2024-04-05T03:09:39.365131+0000","last_peered":"2024-04-05T03:09:39.365131+0000","last_clean":"2024-04-05T03:09:39.365131+0000","last_became_active":"2024-04-05T03:09:04.077114+0000","last_became_peered":"2024-04-05T03:09:04.077114+0000","last_unstale":"2024-04-05T03:09:39.365131+0000","last_undegraded":"2024-04-05T03:09:39.365131+0000","last_fullsized":"2024-04-05T03:09:39.365131+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:52:19.992120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":105,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378107+0000","last_change":"2024-04-05T02:49:47.188521+0000","last_active":"2024-04-05T03:09:38.378107+0000","last_peered":"2024-04-05T03:09:38.378107+0000","last_clean":"2024-04-05T03:09:38.378107+0000","last_became_active":"2024-04-05T02:49:47.188231+0000","last_became_peered":"2024-04-05T02:49:47.188231+0000","last_unstale":"2024-04-05T03:09:38.378107+0000","last_undegraded":"2024-04-05T03:09:38.378107+0000","last_fullsized":"2024-04-05T03:09:38.378107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:49:13.887883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":111,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.369897+0000","last_change":"2024-04-05T03:04:44.542607+0000","last_active":"2024-04-05T03:09:39.369897+0000","last_peered":"2024-04-05T03:09:39.369897+0000","last_clean":"2024-04-05T03:09:39.369897+0000","last_became_active":"2024-04-05T03:04:44.542222+0000","last_became_peered":"2024-04-05T03:04:44.542222+0000","last_unstale":"2024-04-05T03:09:39.369897+0000","last_undegraded":"2024-04-05T03:09:39.369897+0000","last_fullsized":"2024-04-05T03:09:39.369897+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:27:06.813360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.374835+0000","last_change":"2024-04-05T03:09:04.085310+0000","last_active":"2024-04-05T03:09:38.374835+0000","last_peered":"2024-04-05T03:09:38.374835+0000","last_clean":"2024-04-05T03:09:38.374835+0000","last_became_active":"2024-04-05T03:09:04.084911+0000","last_became_peered":"2024-04-05T03:09:04.084911+0000","last_unstale":"2024-04-05T03:09:38.374835+0000","last_undegraded":"2024-04-05T03:09:38.374835+0000","last_fullsized":"2024-04-05T03:09:38.374835+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:23:30.010807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.372832+0000","last_change":"2024-04-05T03:09:39.372832+0000","last_active":"2024-04-05T03:09:39.372832+0000","last_peered":"2024-04-05T03:09:39.372832+0000","last_clean":"2024-04-05T03:09:39.372832+0000","last_became_active":"2024-04-05T03:09:39.372495+0000","last_became_peered":"2024-04-05T03:09:39.372495+0000","last_unstale":"2024-04-05T03:09:39.372832+0000","last_undegraded":"2024-04-05T03:09:39.372832+0000","last_fullsized":"2024-04-05T03:09:39.372832+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:29:22.516358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.384218+0000","last_change":"2024-04-05T03:09:39.383360+0000","last_active":"2024-04-05T03:09:39.384218+0000","last_peered":"2024-04-05T03:09:39.384218+0000","last_clean":"2024-04-05T03:09:39.384218+0000","last_became_active":"2024-04-05T03:09:39.383082+0000","last_became_peered":"2024-04-05T03:09:39.383082+0000","last_unstale":"2024-04-05T03:09:39.384218+0000","last_undegraded":"2024-04-05T03:09:39.384218+0000","last_fullsized":"2024-04-05T03:09:39.384218+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:43.737861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378196+0000","last_change":"2024-04-05T03:09:39.378196+0000","last_active":"2024-04-05T03:09:39.378196+0000","last_peered":"2024-04-05T03:09:39.378196+0000","last_clean":"2024-04-05T03:09:39.378196+0000","last_became_active":"2024-04-05T03:09:39.377747+0000","last_became_peered":"2024-04-05T03:09:39.377747+0000","last_unstale":"2024-04-05T03:09:39.378196+0000","last_undegraded":"2024-04-05T03:09:39.378196+0000","last_fullsized":"2024-04-05T03:09:39.378196+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:54:36.085300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.374414+0000","last_change":"2024-04-05T02:49:47.190900+0000","last_active":"2024-04-05T03:09:38.374414+0000","last_peered":"2024-04-05T03:09:38.374414+0000","last_clean":"2024-04-05T03:09:38.374414+0000","last_became_active":"2024-04-05T02:49:47.190633+0000","last_became_peered":"2024-04-05T02:49:47.190633+0000","last_unstale":"2024-04-05T03:09:38.374414+0000","last_undegraded":"2024-04-05T03:09:38.374414+0000","last_fullsized":"2024-04-05T03:09:38.374414+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:59:46.877238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.383612+0000","last_change":"2024-04-05T03:09:39.383612+0000","last_active":"2024-04-05T03:09:39.383612+0000","last_peered":"2024-04-05T03:09:39.383612+0000","last_clean":"2024-04-05T03:09:39.383612+0000","last_became_active":"2024-04-05T03:09:39.383250+0000","last_became_peered":"2024-04-05T03:09:39.383250+0000","last_unstale":"2024-04-05T03:09:39.383612+0000","last_undegraded":"2024-04-05T03:09:39.383612+0000","last_fullsized":"2024-04-05T03:09:39.383612+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:03:06.149535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375540+0000","last_change":"2024-04-05T03:01:16.080569+0000","last_active":"2024-04-05T03:09:38.375540+0000","last_peered":"2024-04-05T03:09:38.375540+0000","last_clean":"2024-04-05T03:09:38.375540+0000","last_became_active":"2024-04-05T03:01:16.080178+0000","last_became_peered":"2024-04-05T03:01:16.080178+0000","last_unstale":"2024-04-05T03:09:38.375540+0000","last_undegraded":"2024-04-05T03:09:38.375540+0000","last_fullsized":"2024-04-05T03:09:38.375540+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:23:44.048998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.381081+0000","last_change":"2024-04-05T03:09:39.381081+0000","last_active":"2024-04-05T03:09:39.381081+0000","last_peered":"2024-04-05T03:09:39.381081+0000","last_clean":"2024-04-05T03:09:39.381081+0000","last_became_active":"2024-04-05T03:09:39.380743+0000","last_became_peered":"2024-04-05T03:09:39.380743+0000","last_unstale":"2024-04-05T03:09:39.381081+0000","last_undegraded":"2024-04-05T03:09:39.381081+0000","last_fullsized":"2024-04-05T03:09:39.381081+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:41:14.569684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376512+0000","last_change":"2024-04-05T03:01:16.075330+0000","last_active":"2024-04-05T03:09:38.376512+0000","last_peered":"2024-04-05T03:09:38.376512+0000","last_clean":"2024-04-05T03:09:38.376512+0000","last_became_active":"2024-04-05T03:01:16.074609+0000","last_became_peered":"2024-04-05T03:01:16.074609+0000","last_unstale":"2024-04-05T03:09:38.376512+0000","last_undegraded":"2024-04-05T03:09:38.376512+0000","last_fullsized":"2024-04-05T03:09:38.376512+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:09:22.059497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371111+0000","last_change":"2024-04-05T02:49:47.246329+0000","last_active":"2024-04-05T03:09:38.371111+0000","last_peered":"2024-04-05T03:09:38.371111+0000","last_clean":"2024-04-05T03:09:38.371111+0000","last_became_active":"2024-04-05T02:49:47.245961+0000","last_became_peered":"2024-04-05T02:49:47.245961+0000","last_unstale":"2024-04-05T03:09:38.371111+0000","last_undegraded":"2024-04-05T03:09:38.371111+0000","last_fullsized":"2024-04-05T03:09:38.371111+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:45:18.083543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.379504+0000","last_change":"2024-04-05T03:04:44.540214+0000","last_active":"2024-04-05T03:09:38.379504+0000","last_peered":"2024-04-05T03:09:38.379504+0000","last_clean":"2024-04-05T03:09:38.379504+0000","last_became_active":"2024-04-05T03:04:44.539528+0000","last_became_peered":"2024-04-05T03:04:44.539528+0000","last_unstale":"2024-04-05T03:09:38.379504+0000","last_undegraded":"2024-04-05T03:09:38.379504+0000","last_fullsized":"2024-04-05T03:09:38.379504+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:04:02.130527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375588+0000","last_change":"2024-04-05T03:09:39.375588+0000","last_active":"2024-04-05T03:09:39.375588+0000","last_peered":"2024-04-05T03:09:39.375588+0000","last_clean":"2024-04-05T03:09:39.375588+0000","last_became_active":"2024-04-05T03:09:39.375274+0000","last_became_peered":"2024-04-05T03:09:39.375274+0000","last_unstale":"2024-04-05T03:09:39.375588+0000","last_undegraded":"2024-04-05T03:09:39.375588+0000","last_fullsized":"2024-04-05T03:09:39.375588+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:55.755897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":83,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375596+0000","last_change":"2024-04-05T02:49:47.188385+0000","last_active":"2024-04-05T03:09:38.375596+0000","last_peered":"2024-04-05T03:09:38.375596+0000","last_clean":"2024-04-05T03:09:38.375596+0000","last_became_active":"2024-04-05T02:49:47.188108+0000","last_became_peered":"2024-04-05T02:49:47.188108+0000","last_unstale":"2024-04-05T03:09:38.375596+0000","last_undegraded":"2024-04-05T03:09:38.375596+0000","last_fullsized":"2024-04-05T03:09:38.375596+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:01.691480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376358+0000","last_change":"2024-04-05T03:09:04.082639+0000","last_active":"2024-04-05T03:09:38.376358+0000","last_peered":"2024-04-05T03:09:38.376358+0000","last_clean":"2024-04-05T03:09:38.376358+0000","last_became_active":"2024-04-05T03:09:04.082305+0000","last_became_peered":"2024-04-05T03:09:04.082305+0000","last_unstale":"2024-04-05T03:09:38.376358+0000","last_undegraded":"2024-04-05T03:09:38.376358+0000","last_fullsized":"2024-04-05T03:09:38.376358+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:05:56.834271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.377583+0000","last_change":"2024-04-05T03:09:39.377583+0000","last_active":"2024-04-05T03:09:39.377583+0000","last_peered":"2024-04-05T03:09:39.377583+0000","last_clean":"2024-04-05T03:09:39.377583+0000","last_became_active":"2024-04-05T03:09:39.377126+0000","last_became_peered":"2024-04-05T03:09:39.377126+0000","last_unstale":"2024-04-05T03:09:39.377583+0000","last_undegraded":"2024-04-05T03:09:39.377583+0000","last_fullsized":"2024-04-05T03:09:39.377583+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:52:13.977112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":123,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370388+0000","last_change":"2024-04-05T03:09:04.085731+0000","last_active":"2024-04-05T03:09:39.370388+0000","last_peered":"2024-04-05T03:09:39.370388+0000","last_clean":"2024-04-05T03:09:39.370388+0000","last_became_active":"2024-04-05T03:09:04.085403+0000","last_became_peered":"2024-04-05T03:09:04.085403+0000","last_unstale":"2024-04-05T03:09:39.370388+0000","last_undegraded":"2024-04-05T03:09:39.370388+0000","last_fullsized":"2024-04-05T03:09:39.370388+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:53:35.320552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":69,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375151+0000","last_change":"2024-04-05T03:09:39.375151+0000","last_active":"2024-04-05T03:09:39.375151+0000","last_peered":"2024-04-05T03:09:39.375151+0000","last_clean":"2024-04-05T03:09:39.375151+0000","last_became_active":"2024-04-05T03:09:39.374435+0000","last_became_peered":"2024-04-05T03:09:39.374435+0000","last_unstale":"2024-04-05T03:09:39.375151+0000","last_undegraded":"2024-04-05T03:09:39.375151+0000","last_fullsized":"2024-04-05T03:09:39.375151+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:05:54.407704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":91,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378082+0000","last_change":"2024-04-05T03:00:56.995988+0000","last_active":"2024-04-05T03:09:38.378082+0000","last_peered":"2024-04-05T03:09:38.378082+0000","last_clean":"2024-04-05T03:09:38.378082+0000","last_became_active":"2024-04-05T03:00:56.995547+0000","last_became_peered":"2024-04-05T03:00:56.995547+0000","last_unstale":"2024-04-05T03:09:38.378082+0000","last_undegraded":"2024-04-05T03:09:38.378082+0000","last_fullsized":"2024-04-05T03:09:38.378082+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:05:01.486062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376237+0000","last_change":"2024-04-05T03:04:44.546576+0000","last_active":"2024-04-05T03:09:38.376237+0000","last_peered":"2024-04-05T03:09:38.376237+0000","last_clean":"2024-04-05T03:09:38.376237+0000","last_became_active":"2024-04-05T03:04:44.546055+0000","last_became_peered":"2024-04-05T03:04:44.546055+0000","last_unstale":"2024-04-05T03:09:38.376237+0000","last_undegraded":"2024-04-05T03:09:38.376237+0000","last_fullsized":"2024-04-05T03:09:38.376237+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:10:25.535321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376949+0000","last_change":"2024-04-05T03:09:04.086891+0000","last_active":"2024-04-05T03:09:38.376949+0000","last_peered":"2024-04-05T03:09:38.376949+0000","last_clean":"2024-04-05T03:09:38.376949+0000","last_became_active":"2024-04-05T03:09:04.086580+0000","last_became_peered":"2024-04-05T03:09:04.086580+0000","last_unstale":"2024-04-05T03:09:38.376949+0000","last_undegraded":"2024-04-05T03:09:38.376949+0000","last_fullsized":"2024-04-05T03:09:38.376949+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:48:17.436572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375530+0000","last_change":"2024-04-05T03:09:04.085888+0000","last_active":"2024-04-05T03:09:38.375530+0000","last_peered":"2024-04-05T03:09:38.375530+0000","last_clean":"2024-04-05T03:09:38.375530+0000","last_became_active":"2024-04-05T03:09:04.085508+0000","last_became_peered":"2024-04-05T03:09:04.085508+0000","last_unstale":"2024-04-05T03:09:38.375530+0000","last_undegraded":"2024-04-05T03:09:38.375530+0000","last_fullsized":"2024-04-05T03:09:38.375530+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:30.289558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376900+0000","last_change":"2024-04-05T03:04:44.543905+0000","last_active":"2024-04-05T03:09:38.376900+0000","last_peered":"2024-04-05T03:09:38.376900+0000","last_clean":"2024-04-05T03:09:38.376900+0000","last_became_active":"2024-04-05T03:04:44.543460+0000","last_became_peered":"2024-04-05T03:04:44.543460+0000","last_unstale":"2024-04-05T03:09:38.376900+0000","last_undegraded":"2024-04-05T03:09:38.376900+0000","last_fullsized":"2024-04-05T03:09:38.376900+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:43:15.978397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.381238+0000","last_change":"2024-04-05T03:09:39.381238+0000","last_active":"2024-04-05T03:09:39.381238+0000","last_peered":"2024-04-05T03:09:39.381238+0000","last_clean":"2024-04-05T03:09:39.381238+0000","last_became_active":"2024-04-05T03:09:39.380818+0000","last_became_peered":"2024-04-05T03:09:39.380818+0000","last_unstale":"2024-04-05T03:09:39.381238+0000","last_undegraded":"2024-04-05T03:09:39.381238+0000","last_fullsized":"2024-04-05T03:09:39.381238+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:51.255596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.374769+0000","last_change":"2024-04-05T03:09:39.374769+0000","last_active":"2024-04-05T03:09:39.374769+0000","last_peered":"2024-04-05T03:09:39.374769+0000","last_clean":"2024-04-05T03:09:39.374769+0000","last_became_active":"2024-04-05T03:09:39.374419+0000","last_became_peered":"2024-04-05T03:09:39.374419+0000","last_unstale":"2024-04-05T03:09:39.374769+0000","last_undegraded":"2024-04-05T03:09:39.374769+0000","last_fullsized":"2024-04-05T03:09:39.374769+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:57:39.228675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":114,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.367273+0000","last_change":"2024-04-05T03:04:44.541434+0000","last_active":"2024-04-05T03:09:39.367273+0000","last_peered":"2024-04-05T03:09:39.367273+0000","last_clean":"2024-04-05T03:09:39.367273+0000","last_became_active":"2024-04-05T03:04:44.540576+0000","last_became_peered":"2024-04-05T03:04:44.540576+0000","last_unstale":"2024-04-05T03:09:39.367273+0000","last_undegraded":"2024-04-05T03:09:39.367273+0000","last_fullsized":"2024-04-05T03:09:39.367273+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:18:23.831642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378352+0000","last_change":"2024-04-05T03:09:39.378352+0000","last_active":"2024-04-05T03:09:39.378352+0000","last_peered":"2024-04-05T03:09:39.378352+0000","last_clean":"2024-04-05T03:09:39.378352+0000","last_became_active":"2024-04-05T03:09:39.377841+0000","last_became_peered":"2024-04-05T03:09:39.377841+0000","last_unstale":"2024-04-05T03:09:39.378352+0000","last_undegraded":"2024-04-05T03:09:39.378352+0000","last_fullsized":"2024-04-05T03:09:39.378352+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:36.487420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.365236+0000","last_change":"2024-04-05T03:04:44.542586+0000","last_active":"2024-04-05T03:09:39.365236+0000","last_peered":"2024-04-05T03:09:39.365236+0000","last_clean":"2024-04-05T03:09:39.365236+0000","last_became_active":"2024-04-05T03:04:44.542287+0000","last_became_peered":"2024-04-05T03:04:44.542287+0000","last_unstale":"2024-04-05T03:09:39.365236+0000","last_undegraded":"2024-04-05T03:09:39.365236+0000","last_fullsized":"2024-04-05T03:09:39.365236+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:51:19.461316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371674+0000","last_change":"2024-04-05T03:04:44.543040+0000","last_active":"2024-04-05T03:09:38.371674+0000","last_peered":"2024-04-05T03:09:38.371674+0000","last_clean":"2024-04-05T03:09:38.371674+0000","last_became_active":"2024-04-05T03:04:44.542544+0000","last_became_peered":"2024-04-05T03:04:44.542544+0000","last_unstale":"2024-04-05T03:09:38.371674+0000","last_undegraded":"2024-04-05T03:09:38.371674+0000","last_fullsized":"2024-04-05T03:09:38.371674+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:50:56.201624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.377352+0000","last_change":"2024-04-05T03:01:16.083949+0000","last_active":"2024-04-05T03:09:38.377352+0000","last_peered":"2024-04-05T03:09:38.377352+0000","last_clean":"2024-04-05T03:09:38.377352+0000","last_became_active":"2024-04-05T03:01:16.083620+0000","last_became_peered":"2024-04-05T03:01:16.083620+0000","last_unstale":"2024-04-05T03:09:38.377352+0000","last_undegraded":"2024-04-05T03:09:38.377352+0000","last_fullsized":"2024-04-05T03:09:38.377352+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:46:09.105511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375985+0000","last_change":"2024-04-05T03:09:39.375985+0000","last_active":"2024-04-05T03:09:39.375985+0000","last_peered":"2024-04-05T03:09:39.375985+0000","last_clean":"2024-04-05T03:09:39.375985+0000","last_became_active":"2024-04-05T03:09:39.375631+0000","last_became_peered":"2024-04-05T03:09:39.375631+0000","last_unstale":"2024-04-05T03:09:39.375985+0000","last_undegraded":"2024-04-05T03:09:39.375985+0000","last_fullsized":"2024-04-05T03:09:39.375985+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:59:45.236692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.374696+0000","last_change":"2024-04-05T03:09:39.374696+0000","last_active":"2024-04-05T03:09:39.374696+0000","last_peered":"2024-04-05T03:09:39.374696+0000","last_clean":"2024-04-05T03:09:39.374696+0000","last_became_active":"2024-04-05T03:09:39.374352+0000","last_became_peered":"2024-04-05T03:09:39.374352+0000","last_unstale":"2024-04-05T03:09:39.374696+0000","last_undegraded":"2024-04-05T03:09:39.374696+0000","last_fullsized":"2024-04-05T03:09:39.374696+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:21:40.012709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.367241+0000","last_change":"2024-04-05T03:09:04.077384+0000","last_active":"2024-04-05T03:09:39.367241+0000","last_peered":"2024-04-05T03:09:39.367241+0000","last_clean":"2024-04-05T03:09:39.367241+0000","last_became_active":"2024-04-05T03:09:04.077061+0000","last_became_peered":"2024-04-05T03:09:04.077061+0000","last_unstale":"2024-04-05T03:09:39.367241+0000","last_undegraded":"2024-04-05T03:09:39.367241+0000","last_fullsized":"2024-04-05T03:09:39.367241+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:27:15.825689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.363320+0000","last_change":"2024-04-05T03:09:04.082876+0000","last_active":"2024-04-05T03:09:39.363320+0000","last_peered":"2024-04-05T03:09:39.363320+0000","last_clean":"2024-04-05T03:09:39.363320+0000","last_became_active":"2024-04-05T03:09:04.082388+0000","last_became_peered":"2024-04-05T03:09:04.082388+0000","last_unstale":"2024-04-05T03:09:39.363320+0000","last_undegraded":"2024-04-05T03:09:39.363320+0000","last_fullsized":"2024-04-05T03:09:39.363320+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:38:37.187964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371440+0000","last_change":"2024-04-05T03:09:04.085948+0000","last_active":"2024-04-05T03:09:38.371440+0000","last_peered":"2024-04-05T03:09:38.371440+0000","last_clean":"2024-04-05T03:09:38.371440+0000","last_became_active":"2024-04-05T03:09:04.085552+0000","last_became_peered":"2024-04-05T03:09:04.085552+0000","last_unstale":"2024-04-05T03:09:38.371440+0000","last_undegraded":"2024-04-05T03:09:38.371440+0000","last_fullsized":"2024-04-05T03:09:38.371440+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:01:26.532516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"83'161","reported_seq":475,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375133+0000","last_change":"2024-04-05T03:01:16.075358+0000","last_active":"2024-04-05T03:09:38.375133+0000","last_peered":"2024-04-05T03:09:38.375133+0000","last_clean":"2024-04-05T03:09:38.375133+0000","last_became_active":"2024-04-05T03:01:16.074461+0000","last_became_peered":"2024-04-05T03:01:16.074461+0000","last_unstale":"2024-04-05T03:09:38.375133+0000","last_undegraded":"2024-04-05T03:09:38.375133+0000","last_fullsized":"2024-04-05T03:09:38.375133+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_clean_scrub_stamp":"2024-04-05T02:45:59.919732+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:29:13.837561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373822+0000","last_change":"2024-04-05T03:09:39.373822+0000","last_active":"2024-04-05T03:09:39.373822+0000","last_peered":"2024-04-05T03:09:39.373822+0000","last_clean":"2024-04-05T03:09:39.373822+0000","last_became_active":"2024-04-05T03:09:39.373497+0000","last_became_peered":"2024-04-05T03:09:39.373497+0000","last_unstale":"2024-04-05T03:09:39.373822+0000","last_undegraded":"2024-04-05T03:09:39.373822+0000","last_fullsized":"2024-04-05T03:09:39.373822+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:13:36.086162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.377105+0000","last_change":"2024-04-05T02:49:47.246904+0000","last_active":"2024-04-05T03:09:38.377105+0000","last_peered":"2024-04-05T03:09:38.377105+0000","last_clean":"2024-04-05T03:09:38.377105+0000","last_became_active":"2024-04-05T02:49:47.246606+0000","last_became_peered":"2024-04-05T02:49:47.246606+0000","last_unstale":"2024-04-05T03:09:38.377105+0000","last_undegraded":"2024-04-05T03:09:38.377105+0000","last_fullsized":"2024-04-05T03:09:38.377105+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:18:33.167051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375029+0000","last_change":"2024-04-05T03:04:44.541482+0000","last_active":"2024-04-05T03:09:38.375029+0000","last_peered":"2024-04-05T03:09:38.375029+0000","last_clean":"2024-04-05T03:09:38.375029+0000","last_became_active":"2024-04-05T03:04:44.540916+0000","last_became_peered":"2024-04-05T03:04:44.540916+0000","last_unstale":"2024-04-05T03:09:38.375029+0000","last_undegraded":"2024-04-05T03:09:38.375029+0000","last_fullsized":"2024-04-05T03:09:38.375029+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:29:20.609489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378458+0000","last_change":"2024-04-05T03:09:39.378458+0000","last_active":"2024-04-05T03:09:39.378458+0000","last_peered":"2024-04-05T03:09:39.378458+0000","last_clean":"2024-04-05T03:09:39.378458+0000","last_became_active":"2024-04-05T03:09:39.378130+0000","last_became_peered":"2024-04-05T03:09:39.378130+0000","last_unstale":"2024-04-05T03:09:39.378458+0000","last_undegraded":"2024-04-05T03:09:39.378458+0000","last_fullsized":"2024-04-05T03:09:39.378458+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:02:45.828552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":135,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.371349+0000","last_change":"2024-04-05T03:09:04.080636+0000","last_active":"2024-04-05T03:09:39.371349+0000","last_peered":"2024-04-05T03:09:39.371349+0000","last_clean":"2024-04-05T03:09:39.371349+0000","last_became_active":"2024-04-05T03:09:04.080311+0000","last_became_peered":"2024-04-05T03:09:04.080311+0000","last_unstale":"2024-04-05T03:09:39.371349+0000","last_undegraded":"2024-04-05T03:09:39.371349+0000","last_fullsized":"2024-04-05T03:09:39.371349+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:36:14.157698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375663+0000","last_change":"2024-04-05T03:01:16.154762+0000","last_active":"2024-04-05T03:09:38.375663+0000","last_peered":"2024-04-05T03:09:38.375663+0000","last_clean":"2024-04-05T03:09:38.375663+0000","last_became_active":"2024-04-05T03:01:16.154366+0000","last_became_peered":"2024-04-05T03:01:16.154366+0000","last_unstale":"2024-04-05T03:09:38.375663+0000","last_undegraded":"2024-04-05T03:09:38.375663+0000","last_fullsized":"2024-04-05T03:09:38.375663+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:06:30.538880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.367171+0000","last_change":"2024-04-05T03:09:04.083012+0000","last_active":"2024-04-05T03:09:39.367171+0000","last_peered":"2024-04-05T03:09:39.367171+0000","last_clean":"2024-04-05T03:09:39.367171+0000","last_became_active":"2024-04-05T03:09:04.082678+0000","last_became_peered":"2024-04-05T03:09:04.082678+0000","last_unstale":"2024-04-05T03:09:39.367171+0000","last_undegraded":"2024-04-05T03:09:39.367171+0000","last_fullsized":"2024-04-05T03:09:39.367171+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:51:09.230432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":117,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370062+0000","last_change":"2024-04-05T03:01:16.081559+0000","last_active":"2024-04-05T03:09:39.370062+0000","last_peered":"2024-04-05T03:09:39.370062+0000","last_clean":"2024-04-05T03:09:39.370062+0000","last_became_active":"2024-04-05T03:01:16.077624+0000","last_became_peered":"2024-04-05T03:01:16.077624+0000","last_unstale":"2024-04-05T03:09:39.370062+0000","last_undegraded":"2024-04-05T03:09:39.370062+0000","last_fullsized":"2024-04-05T03:09:39.370062+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:23:01.598037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":69,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.376691+0000","last_change":"2024-04-05T03:09:39.376691+0000","last_active":"2024-04-05T03:09:39.376691+0000","last_peered":"2024-04-05T03:09:39.376691+0000","last_clean":"2024-04-05T03:09:39.376691+0000","last_became_active":"2024-04-05T03:09:39.376282+0000","last_became_peered":"2024-04-05T03:09:39.376282+0000","last_unstale":"2024-04-05T03:09:39.376691+0000","last_undegraded":"2024-04-05T03:09:39.376691+0000","last_fullsized":"2024-04-05T03:09:39.376691+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:29:43.628414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":106,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.365888+0000","last_change":"2024-04-05T03:04:44.542358+0000","last_active":"2024-04-05T03:09:39.365888+0000","last_peered":"2024-04-05T03:09:39.365888+0000","last_clean":"2024-04-05T03:09:39.365888+0000","last_became_active":"2024-04-05T03:04:44.540685+0000","last_became_peered":"2024-04-05T03:04:44.540685+0000","last_unstale":"2024-04-05T03:09:39.365888+0000","last_undegraded":"2024-04-05T03:09:39.365888+0000","last_fullsized":"2024-04-05T03:09:39.365888+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:06:49.360769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":73,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371105+0000","last_change":"2024-04-05T03:01:17.081782+0000","last_active":"2024-04-05T03:09:38.371105+0000","last_peered":"2024-04-05T03:09:38.371105+0000","last_clean":"2024-04-05T03:09:38.371105+0000","last_became_active":"2024-04-05T03:01:17.080953+0000","last_became_peered":"2024-04-05T03:01:17.080953+0000","last_unstale":"2024-04-05T03:09:38.371105+0000","last_undegraded":"2024-04-05T03:09:38.371105+0000","last_fullsized":"2024-04-05T03:09:38.371105+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:39.605395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":104,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378269+0000","last_change":"2024-04-05T03:09:04.081811+0000","last_active":"2024-04-05T03:09:38.378269+0000","last_peered":"2024-04-05T03:09:38.378269+0000","last_clean":"2024-04-05T03:09:38.378269+0000","last_became_active":"2024-04-05T03:09:04.081494+0000","last_became_peered":"2024-04-05T03:09:04.081494+0000","last_unstale":"2024-04-05T03:09:38.378269+0000","last_undegraded":"2024-04-05T03:09:38.378269+0000","last_fullsized":"2024-04-05T03:09:38.378269+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:31:27.400390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":129,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370940+0000","last_change":"2024-04-05T03:04:44.544595+0000","last_active":"2024-04-05T03:09:39.370940+0000","last_peered":"2024-04-05T03:09:39.370940+0000","last_clean":"2024-04-05T03:09:39.370940+0000","last_became_active":"2024-04-05T03:04:44.543697+0000","last_became_peered":"2024-04-05T03:04:44.543697+0000","last_unstale":"2024-04-05T03:09:39.370940+0000","last_undegraded":"2024-04-05T03:09:39.370940+0000","last_fullsized":"2024-04-05T03:09:39.370940+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:09:19.091552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370969+0000","last_change":"2024-04-05T03:01:17.081314+0000","last_active":"2024-04-05T03:09:39.370969+0000","last_peered":"2024-04-05T03:09:39.370969+0000","last_clean":"2024-04-05T03:09:39.370969+0000","last_became_active":"2024-04-05T03:01:17.080933+0000","last_became_peered":"2024-04-05T03:01:17.080933+0000","last_unstale":"2024-04-05T03:09:39.370969+0000","last_undegraded":"2024-04-05T03:09:39.370969+0000","last_fullsized":"2024-04-05T03:09:39.370969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:23.984197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":119,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370448+0000","last_change":"2024-04-05T03:01:16.075975+0000","last_active":"2024-04-05T03:09:39.370448+0000","last_peered":"2024-04-05T03:09:39.370448+0000","last_clean":"2024-04-05T03:09:39.370448+0000","last_became_active":"2024-04-05T03:01:16.075414+0000","last_became_peered":"2024-04-05T03:01:16.075414+0000","last_unstale":"2024-04-05T03:09:39.370448+0000","last_undegraded":"2024-04-05T03:09:39.370448+0000","last_fullsized":"2024-04-05T03:09:39.370448+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:27:38.222255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.362574+0000","last_change":"2024-04-05T03:09:04.083841+0000","last_active":"2024-04-05T03:09:39.362574+0000","last_peered":"2024-04-05T03:09:39.362574+0000","last_clean":"2024-04-05T03:09:39.362574+0000","last_became_active":"2024-04-05T03:09:04.083518+0000","last_became_peered":"2024-04-05T03:09:04.083518+0000","last_unstale":"2024-04-05T03:09:39.362574+0000","last_undegraded":"2024-04-05T03:09:39.362574+0000","last_fullsized":"2024-04-05T03:09:39.362574+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:43:45.171084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371831+0000","last_change":"2024-04-05T03:04:44.545489+0000","last_active":"2024-04-05T03:09:38.371831+0000","last_peered":"2024-04-05T03:09:38.371831+0000","last_clean":"2024-04-05T03:09:38.371831+0000","last_became_active":"2024-04-05T03:04:44.545029+0000","last_became_peered":"2024-04-05T03:04:44.545029+0000","last_unstale":"2024-04-05T03:09:38.371831+0000","last_undegraded":"2024-04-05T03:09:38.371831+0000","last_fullsized":"2024-04-05T03:09:38.371831+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:46:39.553004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.377407+0000","last_change":"2024-04-05T03:04:44.545251+0000","last_active":"2024-04-05T03:09:38.377407+0000","last_peered":"2024-04-05T03:09:38.377407+0000","last_clean":"2024-04-05T03:09:38.377407+0000","last_became_active":"2024-04-05T03:04:44.544613+0000","last_became_peered":"2024-04-05T03:04:44.544613+0000","last_unstale":"2024-04-05T03:09:38.377407+0000","last_undegraded":"2024-04-05T03:09:38.377407+0000","last_fullsized":"2024-04-05T03:09:38.377407+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:44:58.655232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":84,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.372376+0000","last_change":"2024-04-05T02:49:47.188962+0000","last_active":"2024-04-05T03:09:38.372376+0000","last_peered":"2024-04-05T03:09:38.372376+0000","last_clean":"2024-04-05T03:09:38.372376+0000","last_became_active":"2024-04-05T02:49:47.188681+0000","last_became_peered":"2024-04-05T02:49:47.188681+0000","last_unstale":"2024-04-05T03:09:38.372376+0000","last_undegraded":"2024-04-05T03:09:38.372376+0000","last_fullsized":"2024-04-05T03:09:38.372376+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:59:25.089268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375048+0000","last_change":"2024-04-05T03:09:39.375048+0000","last_active":"2024-04-05T03:09:39.375048+0000","last_peered":"2024-04-05T03:09:39.375048+0000","last_clean":"2024-04-05T03:09:39.375048+0000","last_became_active":"2024-04-05T03:09:39.374676+0000","last_became_peered":"2024-04-05T03:09:39.374676+0000","last_unstale":"2024-04-05T03:09:39.375048+0000","last_undegraded":"2024-04-05T03:09:39.375048+0000","last_fullsized":"2024-04-05T03:09:39.375048+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:42:49.965978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373194+0000","last_change":"2024-04-05T03:09:39.373194+0000","last_active":"2024-04-05T03:09:39.373194+0000","last_peered":"2024-04-05T03:09:39.373194+0000","last_clean":"2024-04-05T03:09:39.373194+0000","last_became_active":"2024-04-05T03:09:39.372713+0000","last_became_peered":"2024-04-05T03:09:39.372713+0000","last_unstale":"2024-04-05T03:09:39.373194+0000","last_undegraded":"2024-04-05T03:09:39.373194+0000","last_fullsized":"2024-04-05T03:09:39.373194+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:08:24.474831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.376476+0000","last_change":"2024-04-05T03:09:39.376476+0000","last_active":"2024-04-05T03:09:39.376476+0000","last_peered":"2024-04-05T03:09:39.376476+0000","last_clean":"2024-04-05T03:09:39.376476+0000","last_became_active":"2024-04-05T03:09:39.376084+0000","last_became_peered":"2024-04-05T03:09:39.376084+0000","last_unstale":"2024-04-05T03:09:39.376476+0000","last_undegraded":"2024-04-05T03:09:39.376476+0000","last_fullsized":"2024-04-05T03:09:39.376476+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:48:16.648435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":129,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370618+0000","last_change":"2024-04-05T03:09:04.082832+0000","last_active":"2024-04-05T03:09:39.370618+0000","last_peered":"2024-04-05T03:09:39.370618+0000","last_clean":"2024-04-05T03:09:39.370618+0000","last_became_active":"2024-04-05T03:09:04.082444+0000","last_became_peered":"2024-04-05T03:09:04.082444+0000","last_unstale":"2024-04-05T03:09:39.370618+0000","last_undegraded":"2024-04-05T03:09:39.370618+0000","last_fullsized":"2024-04-05T03:09:39.370618+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:20:04.975531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.372573+0000","last_change":"2024-04-05T03:04:44.540054+0000","last_active":"2024-04-05T03:09:38.372573+0000","last_peered":"2024-04-05T03:09:38.372573+0000","last_clean":"2024-04-05T03:09:38.372573+0000","last_became_active":"2024-04-05T03:04:44.539734+0000","last_became_peered":"2024-04-05T03:04:44.539734+0000","last_unstale":"2024-04-05T03:09:38.372573+0000","last_undegraded":"2024-04-05T03:09:38.372573+0000","last_fullsized":"2024-04-05T03:09:38.372573+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:32:59.561866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13599,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201554,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311728,"kb_used_data":2084,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433424,"statfs":{"total":95995035648,"available":95675826176,"internally_reserved":0,"allocated":2134016,"data_stored":1476078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463192,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299428,"kb_used_data":2072,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445724,"statfs":{"total":95995035648,"available":95688421376,"internally_reserved":0,"allocated":2121728,"data_stored":1475932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":119,"seq":511101108296,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":229228,"kb_used_data":1560,"kb_used_omap":5,"kb_used_meta":227642,"kb_avail":93515924,"statfs":{"total":95995035648,"available":95760306176,"internally_reserved":0,"allocated":1597440,"data_stored":893673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":233105687},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925938,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":248740,"kb_used_data":2192,"kb_used_omap":5,"kb_used_meta":246522,"kb_avail":93496412,"statfs":{"total":95995035648,"available":95740325888,"internally_reserved":0,"allocated":2244608,"data_stored":1487160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6054,"internal_metadata":252438618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":134,"seq":575525617685,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":52872,"kb_used_data":1524,"kb_used_omap":6,"kb_used_meta":51321,"kb_avail":93692280,"statfs":{"total":95995035648,"available":95940894720,"internally_reserved":0,"allocated":1560576,"data_stored":888902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6668,"internal_metadata":52553204},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":139,"seq":597000454157,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":78376,"kb_used_data":1492,"kb_used_omap":4,"kb_used_meta":76859,"kb_avail":93666776,"statfs":{"total":95995035648,"available":95914778624,"internally_reserved":0,"allocated":1527808,"data_stored":884112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4639,"internal_metadata":78704097},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986465,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587240,"kb_used_data":1500,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":93157912,"statfs":{"total":95995035648,"available":95393701888,"internally_reserved":0,"allocated":1536000,"data_stored":885640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724827,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599512,"kb_used_data":1476,"kb_used_omap":1,"kb_used_meta":598014,"kb_avail":93145640,"statfs":{"total":95995035648,"available":95381135360,"internally_reserved":0,"allocated":1511424,"data_stored":880802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612366680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4350,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2153,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2935,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:10:38.893 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-05T03:10:39.149 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:39.534 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:39 smithi150 ceph-mon[103445]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:10:39.535 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:39 smithi150 ceph-mon[107701]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:10:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:39 smithi196 ceph-mon[91352]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:10:40.274 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:40 smithi150 ceph-mon[103445]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:40.274 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:40 smithi150 ceph-mon[107701]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:40.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:40 smithi196 ceph-mon[91352]: pgmap v524: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:40.968 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:10:40.970 INFO:teuthology.orchestra.run.smithi150.stderr:dumped all 2024-04-05T03:10:41.521 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:41.522 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:41.573 INFO:teuthology.orchestra.run.smithi150.stdout:{"pg_ready":true,"pg_map":{"version":524,"stamp":"2024-04-05T03:10:39.117838+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595898,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":301,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":217,"ondisk_log_size":217,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2407124,"kb_used_data":13900,"kb_used_omap":29,"kb_used_meta":2393058,"kb_avail":747554092,"statfs":{"total":767960285184,"available":765495390208,"internally_reserved":0,"allocated":14233600,"data_stored":8872299,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30042,"internal_metadata":2450492070},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002610"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":117,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376505+0000","last_change":"2024-04-05T03:09:04.085323+0000","last_active":"2024-04-05T03:09:38.376505+0000","last_peered":"2024-04-05T03:09:38.376505+0000","last_clean":"2024-04-05T03:09:38.376505+0000","last_became_active":"2024-04-05T03:09:04.084948+0000","last_became_peered":"2024-04-05T03:09:04.084948+0000","last_unstale":"2024-04-05T03:09:38.376505+0000","last_undegraded":"2024-04-05T03:09:38.376505+0000","last_fullsized":"2024-04-05T03:09:38.376505+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:20:46.064821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373362+0000","last_change":"2024-04-05T03:09:39.373362+0000","last_active":"2024-04-05T03:09:39.373362+0000","last_peered":"2024-04-05T03:09:39.373362+0000","last_clean":"2024-04-05T03:09:39.373362+0000","last_became_active":"2024-04-05T03:09:39.372920+0000","last_became_peered":"2024-04-05T03:09:39.372920+0000","last_unstale":"2024-04-05T03:09:39.373362+0000","last_undegraded":"2024-04-05T03:09:39.373362+0000","last_fullsized":"2024-04-05T03:09:39.373362+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:42:38.110269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.379935+0000","last_change":"2024-04-05T03:09:39.379935+0000","last_active":"2024-04-05T03:09:39.379935+0000","last_peered":"2024-04-05T03:09:39.379935+0000","last_clean":"2024-04-05T03:09:39.379935+0000","last_became_active":"2024-04-05T03:09:39.379596+0000","last_became_peered":"2024-04-05T03:09:39.379596+0000","last_unstale":"2024-04-05T03:09:39.379935+0000","last_undegraded":"2024-04-05T03:09:39.379935+0000","last_fullsized":"2024-04-05T03:09:39.379935+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:09:51.368120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":126,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.383940+0000","last_change":"2024-04-05T03:09:39.383940+0000","last_active":"2024-04-05T03:09:39.383940+0000","last_peered":"2024-04-05T03:09:39.383940+0000","last_clean":"2024-04-05T03:09:39.383940+0000","last_became_active":"2024-04-05T03:09:39.383631+0000","last_became_peered":"2024-04-05T03:09:39.383631+0000","last_unstale":"2024-04-05T03:09:39.383940+0000","last_undegraded":"2024-04-05T03:09:39.383940+0000","last_fullsized":"2024-04-05T03:09:39.383940+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:21:33.901632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":109,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.374442+0000","last_change":"2024-04-05T03:09:39.374442+0000","last_active":"2024-04-05T03:09:39.374442+0000","last_peered":"2024-04-05T03:09:39.374442+0000","last_clean":"2024-04-05T03:09:39.374442+0000","last_became_active":"2024-04-05T03:09:39.374060+0000","last_became_peered":"2024-04-05T03:09:39.374060+0000","last_unstale":"2024-04-05T03:09:39.374442+0000","last_undegraded":"2024-04-05T03:09:39.374442+0000","last_fullsized":"2024-04-05T03:09:39.374442+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:02:15.911109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378190+0000","last_change":"2024-04-05T03:04:44.547518+0000","last_active":"2024-04-05T03:09:38.378190+0000","last_peered":"2024-04-05T03:09:38.378190+0000","last_clean":"2024-04-05T03:09:38.378190+0000","last_became_active":"2024-04-05T03:04:44.547154+0000","last_became_peered":"2024-04-05T03:04:44.547154+0000","last_unstale":"2024-04-05T03:09:38.378190+0000","last_undegraded":"2024-04-05T03:09:38.378190+0000","last_fullsized":"2024-04-05T03:09:38.378190+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:25:14.595296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":124,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378073+0000","last_change":"2024-04-05T03:09:39.378073+0000","last_active":"2024-04-05T03:09:39.378073+0000","last_peered":"2024-04-05T03:09:39.378073+0000","last_clean":"2024-04-05T03:09:39.378073+0000","last_became_active":"2024-04-05T03:09:39.377691+0000","last_became_peered":"2024-04-05T03:09:39.377691+0000","last_unstale":"2024-04-05T03:09:39.378073+0000","last_undegraded":"2024-04-05T03:09:39.378073+0000","last_fullsized":"2024-04-05T03:09:39.378073+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:00:01.207793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373884+0000","last_change":"2024-04-05T03:09:39.373884+0000","last_active":"2024-04-05T03:09:39.373884+0000","last_peered":"2024-04-05T03:09:39.373884+0000","last_clean":"2024-04-05T03:09:39.373884+0000","last_became_active":"2024-04-05T03:09:39.373578+0000","last_became_peered":"2024-04-05T03:09:39.373578+0000","last_unstale":"2024-04-05T03:09:39.373884+0000","last_undegraded":"2024-04-05T03:09:39.373884+0000","last_fullsized":"2024-04-05T03:09:39.373884+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:01:50.207471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371704+0000","last_change":"2024-04-05T03:04:44.542000+0000","last_active":"2024-04-05T03:09:38.371704+0000","last_peered":"2024-04-05T03:09:38.371704+0000","last_clean":"2024-04-05T03:09:38.371704+0000","last_became_active":"2024-04-05T03:04:44.541683+0000","last_became_peered":"2024-04-05T03:04:44.541683+0000","last_unstale":"2024-04-05T03:09:38.371704+0000","last_undegraded":"2024-04-05T03:09:38.371704+0000","last_fullsized":"2024-04-05T03:09:38.371704+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:42:28.072974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":64,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.372268+0000","last_change":"2024-04-05T03:09:39.372268+0000","last_active":"2024-04-05T03:09:39.372268+0000","last_peered":"2024-04-05T03:09:39.372268+0000","last_clean":"2024-04-05T03:09:39.372268+0000","last_became_active":"2024-04-05T03:09:39.371920+0000","last_became_peered":"2024-04-05T03:09:39.371920+0000","last_unstale":"2024-04-05T03:09:39.372268+0000","last_undegraded":"2024-04-05T03:09:39.372268+0000","last_fullsized":"2024-04-05T03:09:39.372268+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:31:28.817062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":102,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.377767+0000","last_change":"2024-04-05T03:09:39.377767+0000","last_active":"2024-04-05T03:09:39.377767+0000","last_peered":"2024-04-05T03:09:39.377767+0000","last_clean":"2024-04-05T03:09:39.377767+0000","last_became_active":"2024-04-05T03:09:39.377354+0000","last_became_peered":"2024-04-05T03:09:39.377354+0000","last_unstale":"2024-04-05T03:09:39.377767+0000","last_undegraded":"2024-04-05T03:09:39.377767+0000","last_fullsized":"2024-04-05T03:09:39.377767+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:27:22.024303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.365131+0000","last_change":"2024-04-05T03:09:04.077448+0000","last_active":"2024-04-05T03:09:39.365131+0000","last_peered":"2024-04-05T03:09:39.365131+0000","last_clean":"2024-04-05T03:09:39.365131+0000","last_became_active":"2024-04-05T03:09:04.077114+0000","last_became_peered":"2024-04-05T03:09:04.077114+0000","last_unstale":"2024-04-05T03:09:39.365131+0000","last_undegraded":"2024-04-05T03:09:39.365131+0000","last_fullsized":"2024-04-05T03:09:39.365131+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:52:19.992120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"83'23","reported_seq":105,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378107+0000","last_change":"2024-04-05T02:49:47.188521+0000","last_active":"2024-04-05T03:09:38.378107+0000","last_peered":"2024-04-05T03:09:38.378107+0000","last_clean":"2024-04-05T03:09:38.378107+0000","last_became_active":"2024-04-05T02:49:47.188231+0000","last_became_peered":"2024-04-05T02:49:47.188231+0000","last_unstale":"2024-04-05T03:09:38.378107+0000","last_undegraded":"2024-04-05T03:09:38.378107+0000","last_fullsized":"2024-04-05T03:09:38.378107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:49:13.887883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":111,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.369897+0000","last_change":"2024-04-05T03:04:44.542607+0000","last_active":"2024-04-05T03:09:39.369897+0000","last_peered":"2024-04-05T03:09:39.369897+0000","last_clean":"2024-04-05T03:09:39.369897+0000","last_became_active":"2024-04-05T03:04:44.542222+0000","last_became_peered":"2024-04-05T03:04:44.542222+0000","last_unstale":"2024-04-05T03:09:39.369897+0000","last_undegraded":"2024-04-05T03:09:39.369897+0000","last_fullsized":"2024-04-05T03:09:39.369897+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:27:06.813360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.374835+0000","last_change":"2024-04-05T03:09:04.085310+0000","last_active":"2024-04-05T03:09:38.374835+0000","last_peered":"2024-04-05T03:09:38.374835+0000","last_clean":"2024-04-05T03:09:38.374835+0000","last_became_active":"2024-04-05T03:09:04.084911+0000","last_became_peered":"2024-04-05T03:09:04.084911+0000","last_unstale":"2024-04-05T03:09:38.374835+0000","last_undegraded":"2024-04-05T03:09:38.374835+0000","last_fullsized":"2024-04-05T03:09:38.374835+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:23:30.010807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.372832+0000","last_change":"2024-04-05T03:09:39.372832+0000","last_active":"2024-04-05T03:09:39.372832+0000","last_peered":"2024-04-05T03:09:39.372832+0000","last_clean":"2024-04-05T03:09:39.372832+0000","last_became_active":"2024-04-05T03:09:39.372495+0000","last_became_peered":"2024-04-05T03:09:39.372495+0000","last_unstale":"2024-04-05T03:09:39.372832+0000","last_undegraded":"2024-04-05T03:09:39.372832+0000","last_fullsized":"2024-04-05T03:09:39.372832+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:29:22.516358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.384218+0000","last_change":"2024-04-05T03:09:39.383360+0000","last_active":"2024-04-05T03:09:39.384218+0000","last_peered":"2024-04-05T03:09:39.384218+0000","last_clean":"2024-04-05T03:09:39.384218+0000","last_became_active":"2024-04-05T03:09:39.383082+0000","last_became_peered":"2024-04-05T03:09:39.383082+0000","last_unstale":"2024-04-05T03:09:39.384218+0000","last_undegraded":"2024-04-05T03:09:39.384218+0000","last_fullsized":"2024-04-05T03:09:39.384218+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:16:43.737861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378196+0000","last_change":"2024-04-05T03:09:39.378196+0000","last_active":"2024-04-05T03:09:39.378196+0000","last_peered":"2024-04-05T03:09:39.378196+0000","last_clean":"2024-04-05T03:09:39.378196+0000","last_became_active":"2024-04-05T03:09:39.377747+0000","last_became_peered":"2024-04-05T03:09:39.377747+0000","last_unstale":"2024-04-05T03:09:39.378196+0000","last_undegraded":"2024-04-05T03:09:39.378196+0000","last_fullsized":"2024-04-05T03:09:39.378196+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:54:36.085300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.374414+0000","last_change":"2024-04-05T02:49:47.190900+0000","last_active":"2024-04-05T03:09:38.374414+0000","last_peered":"2024-04-05T03:09:38.374414+0000","last_clean":"2024-04-05T03:09:38.374414+0000","last_became_active":"2024-04-05T02:49:47.190633+0000","last_became_peered":"2024-04-05T02:49:47.190633+0000","last_unstale":"2024-04-05T03:09:38.374414+0000","last_undegraded":"2024-04-05T03:09:38.374414+0000","last_fullsized":"2024-04-05T03:09:38.374414+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:59:46.877238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.383612+0000","last_change":"2024-04-05T03:09:39.383612+0000","last_active":"2024-04-05T03:09:39.383612+0000","last_peered":"2024-04-05T03:09:39.383612+0000","last_clean":"2024-04-05T03:09:39.383612+0000","last_became_active":"2024-04-05T03:09:39.383250+0000","last_became_peered":"2024-04-05T03:09:39.383250+0000","last_unstale":"2024-04-05T03:09:39.383612+0000","last_undegraded":"2024-04-05T03:09:39.383612+0000","last_fullsized":"2024-04-05T03:09:39.383612+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:03:06.149535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375540+0000","last_change":"2024-04-05T03:01:16.080569+0000","last_active":"2024-04-05T03:09:38.375540+0000","last_peered":"2024-04-05T03:09:38.375540+0000","last_clean":"2024-04-05T03:09:38.375540+0000","last_became_active":"2024-04-05T03:01:16.080178+0000","last_became_peered":"2024-04-05T03:01:16.080178+0000","last_unstale":"2024-04-05T03:09:38.375540+0000","last_undegraded":"2024-04-05T03:09:38.375540+0000","last_fullsized":"2024-04-05T03:09:38.375540+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:23:44.048998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.381081+0000","last_change":"2024-04-05T03:09:39.381081+0000","last_active":"2024-04-05T03:09:39.381081+0000","last_peered":"2024-04-05T03:09:39.381081+0000","last_clean":"2024-04-05T03:09:39.381081+0000","last_became_active":"2024-04-05T03:09:39.380743+0000","last_became_peered":"2024-04-05T03:09:39.380743+0000","last_unstale":"2024-04-05T03:09:39.381081+0000","last_undegraded":"2024-04-05T03:09:39.381081+0000","last_fullsized":"2024-04-05T03:09:39.381081+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:41:14.569684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376512+0000","last_change":"2024-04-05T03:01:16.075330+0000","last_active":"2024-04-05T03:09:38.376512+0000","last_peered":"2024-04-05T03:09:38.376512+0000","last_clean":"2024-04-05T03:09:38.376512+0000","last_became_active":"2024-04-05T03:01:16.074609+0000","last_became_peered":"2024-04-05T03:01:16.074609+0000","last_unstale":"2024-04-05T03:09:38.376512+0000","last_undegraded":"2024-04-05T03:09:38.376512+0000","last_fullsized":"2024-04-05T03:09:38.376512+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:09:22.059497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371111+0000","last_change":"2024-04-05T02:49:47.246329+0000","last_active":"2024-04-05T03:09:38.371111+0000","last_peered":"2024-04-05T03:09:38.371111+0000","last_clean":"2024-04-05T03:09:38.371111+0000","last_became_active":"2024-04-05T02:49:47.245961+0000","last_became_peered":"2024-04-05T02:49:47.245961+0000","last_unstale":"2024-04-05T03:09:38.371111+0000","last_undegraded":"2024-04-05T03:09:38.371111+0000","last_fullsized":"2024-04-05T03:09:38.371111+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:45:18.083543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.379504+0000","last_change":"2024-04-05T03:04:44.540214+0000","last_active":"2024-04-05T03:09:38.379504+0000","last_peered":"2024-04-05T03:09:38.379504+0000","last_clean":"2024-04-05T03:09:38.379504+0000","last_became_active":"2024-04-05T03:04:44.539528+0000","last_became_peered":"2024-04-05T03:04:44.539528+0000","last_unstale":"2024-04-05T03:09:38.379504+0000","last_undegraded":"2024-04-05T03:09:38.379504+0000","last_fullsized":"2024-04-05T03:09:38.379504+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:04:02.130527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375588+0000","last_change":"2024-04-05T03:09:39.375588+0000","last_active":"2024-04-05T03:09:39.375588+0000","last_peered":"2024-04-05T03:09:39.375588+0000","last_clean":"2024-04-05T03:09:39.375588+0000","last_became_active":"2024-04-05T03:09:39.375274+0000","last_became_peered":"2024-04-05T03:09:39.375274+0000","last_unstale":"2024-04-05T03:09:39.375588+0000","last_undegraded":"2024-04-05T03:09:39.375588+0000","last_fullsized":"2024-04-05T03:09:39.375588+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:09:55.755897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":83,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375596+0000","last_change":"2024-04-05T02:49:47.188385+0000","last_active":"2024-04-05T03:09:38.375596+0000","last_peered":"2024-04-05T03:09:38.375596+0000","last_clean":"2024-04-05T03:09:38.375596+0000","last_became_active":"2024-04-05T02:49:47.188108+0000","last_became_peered":"2024-04-05T02:49:47.188108+0000","last_unstale":"2024-04-05T03:09:38.375596+0000","last_undegraded":"2024-04-05T03:09:38.375596+0000","last_fullsized":"2024-04-05T03:09:38.375596+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:47:01.691480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376358+0000","last_change":"2024-04-05T03:09:04.082639+0000","last_active":"2024-04-05T03:09:38.376358+0000","last_peered":"2024-04-05T03:09:38.376358+0000","last_clean":"2024-04-05T03:09:38.376358+0000","last_became_active":"2024-04-05T03:09:04.082305+0000","last_became_peered":"2024-04-05T03:09:04.082305+0000","last_unstale":"2024-04-05T03:09:38.376358+0000","last_undegraded":"2024-04-05T03:09:38.376358+0000","last_fullsized":"2024-04-05T03:09:38.376358+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:05:56.834271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.377583+0000","last_change":"2024-04-05T03:09:39.377583+0000","last_active":"2024-04-05T03:09:39.377583+0000","last_peered":"2024-04-05T03:09:39.377583+0000","last_clean":"2024-04-05T03:09:39.377583+0000","last_became_active":"2024-04-05T03:09:39.377126+0000","last_became_peered":"2024-04-05T03:09:39.377126+0000","last_unstale":"2024-04-05T03:09:39.377583+0000","last_undegraded":"2024-04-05T03:09:39.377583+0000","last_fullsized":"2024-04-05T03:09:39.377583+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:52:13.977112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":123,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370388+0000","last_change":"2024-04-05T03:09:04.085731+0000","last_active":"2024-04-05T03:09:39.370388+0000","last_peered":"2024-04-05T03:09:39.370388+0000","last_clean":"2024-04-05T03:09:39.370388+0000","last_became_active":"2024-04-05T03:09:04.085403+0000","last_became_peered":"2024-04-05T03:09:04.085403+0000","last_unstale":"2024-04-05T03:09:39.370388+0000","last_undegraded":"2024-04-05T03:09:39.370388+0000","last_fullsized":"2024-04-05T03:09:39.370388+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:53:35.320552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":69,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375151+0000","last_change":"2024-04-05T03:09:39.375151+0000","last_active":"2024-04-05T03:09:39.375151+0000","last_peered":"2024-04-05T03:09:39.375151+0000","last_clean":"2024-04-05T03:09:39.375151+0000","last_became_active":"2024-04-05T03:09:39.374435+0000","last_became_peered":"2024-04-05T03:09:39.374435+0000","last_unstale":"2024-04-05T03:09:39.375151+0000","last_undegraded":"2024-04-05T03:09:39.375151+0000","last_fullsized":"2024-04-05T03:09:39.375151+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:05:54.407704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":91,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378082+0000","last_change":"2024-04-05T03:00:56.995988+0000","last_active":"2024-04-05T03:09:38.378082+0000","last_peered":"2024-04-05T03:09:38.378082+0000","last_clean":"2024-04-05T03:09:38.378082+0000","last_became_active":"2024-04-05T03:00:56.995547+0000","last_became_peered":"2024-04-05T03:00:56.995547+0000","last_unstale":"2024-04-05T03:09:38.378082+0000","last_undegraded":"2024-04-05T03:09:38.378082+0000","last_fullsized":"2024-04-05T03:09:38.378082+0000","mapping_epoch":90,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":91,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:05:01.486062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376237+0000","last_change":"2024-04-05T03:04:44.546576+0000","last_active":"2024-04-05T03:09:38.376237+0000","last_peered":"2024-04-05T03:09:38.376237+0000","last_clean":"2024-04-05T03:09:38.376237+0000","last_became_active":"2024-04-05T03:04:44.546055+0000","last_became_peered":"2024-04-05T03:04:44.546055+0000","last_unstale":"2024-04-05T03:09:38.376237+0000","last_undegraded":"2024-04-05T03:09:38.376237+0000","last_fullsized":"2024-04-05T03:09:38.376237+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:10:25.535321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376949+0000","last_change":"2024-04-05T03:09:04.086891+0000","last_active":"2024-04-05T03:09:38.376949+0000","last_peered":"2024-04-05T03:09:38.376949+0000","last_clean":"2024-04-05T03:09:38.376949+0000","last_became_active":"2024-04-05T03:09:04.086580+0000","last_became_peered":"2024-04-05T03:09:04.086580+0000","last_unstale":"2024-04-05T03:09:38.376949+0000","last_undegraded":"2024-04-05T03:09:38.376949+0000","last_fullsized":"2024-04-05T03:09:38.376949+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:48:17.436572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375530+0000","last_change":"2024-04-05T03:09:04.085888+0000","last_active":"2024-04-05T03:09:38.375530+0000","last_peered":"2024-04-05T03:09:38.375530+0000","last_clean":"2024-04-05T03:09:38.375530+0000","last_became_active":"2024-04-05T03:09:04.085508+0000","last_became_peered":"2024-04-05T03:09:04.085508+0000","last_unstale":"2024-04-05T03:09:38.375530+0000","last_undegraded":"2024-04-05T03:09:38.375530+0000","last_fullsized":"2024-04-05T03:09:38.375530+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:02:30.289558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.376900+0000","last_change":"2024-04-05T03:04:44.543905+0000","last_active":"2024-04-05T03:09:38.376900+0000","last_peered":"2024-04-05T03:09:38.376900+0000","last_clean":"2024-04-05T03:09:38.376900+0000","last_became_active":"2024-04-05T03:04:44.543460+0000","last_became_peered":"2024-04-05T03:04:44.543460+0000","last_unstale":"2024-04-05T03:09:38.376900+0000","last_undegraded":"2024-04-05T03:09:38.376900+0000","last_fullsized":"2024-04-05T03:09:38.376900+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:43:15.978397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.381238+0000","last_change":"2024-04-05T03:09:39.381238+0000","last_active":"2024-04-05T03:09:39.381238+0000","last_peered":"2024-04-05T03:09:39.381238+0000","last_clean":"2024-04-05T03:09:39.381238+0000","last_became_active":"2024-04-05T03:09:39.380818+0000","last_became_peered":"2024-04-05T03:09:39.380818+0000","last_unstale":"2024-04-05T03:09:39.381238+0000","last_undegraded":"2024-04-05T03:09:39.381238+0000","last_fullsized":"2024-04-05T03:09:39.381238+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:51.255596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.374769+0000","last_change":"2024-04-05T03:09:39.374769+0000","last_active":"2024-04-05T03:09:39.374769+0000","last_peered":"2024-04-05T03:09:39.374769+0000","last_clean":"2024-04-05T03:09:39.374769+0000","last_became_active":"2024-04-05T03:09:39.374419+0000","last_became_peered":"2024-04-05T03:09:39.374419+0000","last_unstale":"2024-04-05T03:09:39.374769+0000","last_undegraded":"2024-04-05T03:09:39.374769+0000","last_fullsized":"2024-04-05T03:09:39.374769+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:57:39.228675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":114,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.367273+0000","last_change":"2024-04-05T03:04:44.541434+0000","last_active":"2024-04-05T03:09:39.367273+0000","last_peered":"2024-04-05T03:09:39.367273+0000","last_clean":"2024-04-05T03:09:39.367273+0000","last_became_active":"2024-04-05T03:04:44.540576+0000","last_became_peered":"2024-04-05T03:04:44.540576+0000","last_unstale":"2024-04-05T03:09:39.367273+0000","last_undegraded":"2024-04-05T03:09:39.367273+0000","last_fullsized":"2024-04-05T03:09:39.367273+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:18:23.831642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378352+0000","last_change":"2024-04-05T03:09:39.378352+0000","last_active":"2024-04-05T03:09:39.378352+0000","last_peered":"2024-04-05T03:09:39.378352+0000","last_clean":"2024-04-05T03:09:39.378352+0000","last_became_active":"2024-04-05T03:09:39.377841+0000","last_became_peered":"2024-04-05T03:09:39.377841+0000","last_unstale":"2024-04-05T03:09:39.378352+0000","last_undegraded":"2024-04-05T03:09:39.378352+0000","last_fullsized":"2024-04-05T03:09:39.378352+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:13:36.487420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.365236+0000","last_change":"2024-04-05T03:04:44.542586+0000","last_active":"2024-04-05T03:09:39.365236+0000","last_peered":"2024-04-05T03:09:39.365236+0000","last_clean":"2024-04-05T03:09:39.365236+0000","last_became_active":"2024-04-05T03:04:44.542287+0000","last_became_peered":"2024-04-05T03:04:44.542287+0000","last_unstale":"2024-04-05T03:09:39.365236+0000","last_undegraded":"2024-04-05T03:09:39.365236+0000","last_fullsized":"2024-04-05T03:09:39.365236+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:51:19.461316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371674+0000","last_change":"2024-04-05T03:04:44.543040+0000","last_active":"2024-04-05T03:09:38.371674+0000","last_peered":"2024-04-05T03:09:38.371674+0000","last_clean":"2024-04-05T03:09:38.371674+0000","last_became_active":"2024-04-05T03:04:44.542544+0000","last_became_peered":"2024-04-05T03:04:44.542544+0000","last_unstale":"2024-04-05T03:09:38.371674+0000","last_undegraded":"2024-04-05T03:09:38.371674+0000","last_fullsized":"2024-04-05T03:09:38.371674+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:50:56.201624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.377352+0000","last_change":"2024-04-05T03:01:16.083949+0000","last_active":"2024-04-05T03:09:38.377352+0000","last_peered":"2024-04-05T03:09:38.377352+0000","last_clean":"2024-04-05T03:09:38.377352+0000","last_became_active":"2024-04-05T03:01:16.083620+0000","last_became_peered":"2024-04-05T03:01:16.083620+0000","last_unstale":"2024-04-05T03:09:38.377352+0000","last_undegraded":"2024-04-05T03:09:38.377352+0000","last_fullsized":"2024-04-05T03:09:38.377352+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:46:09.105511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375985+0000","last_change":"2024-04-05T03:09:39.375985+0000","last_active":"2024-04-05T03:09:39.375985+0000","last_peered":"2024-04-05T03:09:39.375985+0000","last_clean":"2024-04-05T03:09:39.375985+0000","last_became_active":"2024-04-05T03:09:39.375631+0000","last_became_peered":"2024-04-05T03:09:39.375631+0000","last_unstale":"2024-04-05T03:09:39.375985+0000","last_undegraded":"2024-04-05T03:09:39.375985+0000","last_fullsized":"2024-04-05T03:09:39.375985+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:59:45.236692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":91,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.374696+0000","last_change":"2024-04-05T03:09:39.374696+0000","last_active":"2024-04-05T03:09:39.374696+0000","last_peered":"2024-04-05T03:09:39.374696+0000","last_clean":"2024-04-05T03:09:39.374696+0000","last_became_active":"2024-04-05T03:09:39.374352+0000","last_became_peered":"2024-04-05T03:09:39.374352+0000","last_unstale":"2024-04-05T03:09:39.374696+0000","last_undegraded":"2024-04-05T03:09:39.374696+0000","last_fullsized":"2024-04-05T03:09:39.374696+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:21:40.012709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.367241+0000","last_change":"2024-04-05T03:09:04.077384+0000","last_active":"2024-04-05T03:09:39.367241+0000","last_peered":"2024-04-05T03:09:39.367241+0000","last_clean":"2024-04-05T03:09:39.367241+0000","last_became_active":"2024-04-05T03:09:04.077061+0000","last_became_peered":"2024-04-05T03:09:04.077061+0000","last_unstale":"2024-04-05T03:09:39.367241+0000","last_undegraded":"2024-04-05T03:09:39.367241+0000","last_fullsized":"2024-04-05T03:09:39.367241+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:27:15.825689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.363320+0000","last_change":"2024-04-05T03:09:04.082876+0000","last_active":"2024-04-05T03:09:39.363320+0000","last_peered":"2024-04-05T03:09:39.363320+0000","last_clean":"2024-04-05T03:09:39.363320+0000","last_became_active":"2024-04-05T03:09:04.082388+0000","last_became_peered":"2024-04-05T03:09:04.082388+0000","last_unstale":"2024-04-05T03:09:39.363320+0000","last_undegraded":"2024-04-05T03:09:39.363320+0000","last_fullsized":"2024-04-05T03:09:39.363320+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:38:37.187964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371440+0000","last_change":"2024-04-05T03:09:04.085948+0000","last_active":"2024-04-05T03:09:38.371440+0000","last_peered":"2024-04-05T03:09:38.371440+0000","last_clean":"2024-04-05T03:09:38.371440+0000","last_became_active":"2024-04-05T03:09:04.085552+0000","last_became_peered":"2024-04-05T03:09:04.085552+0000","last_unstale":"2024-04-05T03:09:38.371440+0000","last_undegraded":"2024-04-05T03:09:38.371440+0000","last_fullsized":"2024-04-05T03:09:38.371440+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:01:26.532516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"83'161","reported_seq":475,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375133+0000","last_change":"2024-04-05T03:01:16.075358+0000","last_active":"2024-04-05T03:09:38.375133+0000","last_peered":"2024-04-05T03:09:38.375133+0000","last_clean":"2024-04-05T03:09:38.375133+0000","last_became_active":"2024-04-05T03:01:16.074461+0000","last_became_peered":"2024-04-05T03:01:16.074461+0000","last_unstale":"2024-04-05T03:09:38.375133+0000","last_undegraded":"2024-04-05T03:09:38.375133+0000","last_fullsized":"2024-04-05T03:09:38.375133+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:45:59.919732+0000","last_clean_scrub_stamp":"2024-04-05T02:45:59.919732+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:29:13.837561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":63,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373822+0000","last_change":"2024-04-05T03:09:39.373822+0000","last_active":"2024-04-05T03:09:39.373822+0000","last_peered":"2024-04-05T03:09:39.373822+0000","last_clean":"2024-04-05T03:09:39.373822+0000","last_became_active":"2024-04-05T03:09:39.373497+0000","last_became_peered":"2024-04-05T03:09:39.373497+0000","last_unstale":"2024-04-05T03:09:39.373822+0000","last_undegraded":"2024-04-05T03:09:39.373822+0000","last_fullsized":"2024-04-05T03:09:39.373822+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T11:13:36.086162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.377105+0000","last_change":"2024-04-05T02:49:47.246904+0000","last_active":"2024-04-05T03:09:38.377105+0000","last_peered":"2024-04-05T03:09:38.377105+0000","last_clean":"2024-04-05T03:09:38.377105+0000","last_became_active":"2024-04-05T02:49:47.246606+0000","last_became_peered":"2024-04-05T02:49:47.246606+0000","last_unstale":"2024-04-05T03:09:38.377105+0000","last_undegraded":"2024-04-05T03:09:38.377105+0000","last_fullsized":"2024-04-05T03:09:38.377105+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:18:33.167051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":120,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375029+0000","last_change":"2024-04-05T03:04:44.541482+0000","last_active":"2024-04-05T03:09:38.375029+0000","last_peered":"2024-04-05T03:09:38.375029+0000","last_clean":"2024-04-05T03:09:38.375029+0000","last_became_active":"2024-04-05T03:04:44.540916+0000","last_became_peered":"2024-04-05T03:04:44.540916+0000","last_unstale":"2024-04-05T03:09:38.375029+0000","last_undegraded":"2024-04-05T03:09:38.375029+0000","last_fullsized":"2024-04-05T03:09:38.375029+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:29:20.609489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":125,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.378458+0000","last_change":"2024-04-05T03:09:39.378458+0000","last_active":"2024-04-05T03:09:39.378458+0000","last_peered":"2024-04-05T03:09:39.378458+0000","last_clean":"2024-04-05T03:09:39.378458+0000","last_became_active":"2024-04-05T03:09:39.378130+0000","last_became_peered":"2024-04-05T03:09:39.378130+0000","last_unstale":"2024-04-05T03:09:39.378458+0000","last_undegraded":"2024-04-05T03:09:39.378458+0000","last_fullsized":"2024-04-05T03:09:39.378458+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:02:45.828552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":135,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.371349+0000","last_change":"2024-04-05T03:09:04.080636+0000","last_active":"2024-04-05T03:09:39.371349+0000","last_peered":"2024-04-05T03:09:39.371349+0000","last_clean":"2024-04-05T03:09:39.371349+0000","last_became_active":"2024-04-05T03:09:04.080311+0000","last_became_peered":"2024-04-05T03:09:04.080311+0000","last_unstale":"2024-04-05T03:09:39.371349+0000","last_undegraded":"2024-04-05T03:09:39.371349+0000","last_fullsized":"2024-04-05T03:09:39.371349+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:36:14.157698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.375663+0000","last_change":"2024-04-05T03:01:16.154762+0000","last_active":"2024-04-05T03:09:38.375663+0000","last_peered":"2024-04-05T03:09:38.375663+0000","last_clean":"2024-04-05T03:09:38.375663+0000","last_became_active":"2024-04-05T03:01:16.154366+0000","last_became_peered":"2024-04-05T03:01:16.154366+0000","last_unstale":"2024-04-05T03:09:38.375663+0000","last_undegraded":"2024-04-05T03:09:38.375663+0000","last_fullsized":"2024-04-05T03:09:38.375663+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T06:06:30.538880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.367171+0000","last_change":"2024-04-05T03:09:04.083012+0000","last_active":"2024-04-05T03:09:39.367171+0000","last_peered":"2024-04-05T03:09:39.367171+0000","last_clean":"2024-04-05T03:09:39.367171+0000","last_became_active":"2024-04-05T03:09:04.082678+0000","last_became_peered":"2024-04-05T03:09:04.082678+0000","last_unstale":"2024-04-05T03:09:39.367171+0000","last_undegraded":"2024-04-05T03:09:39.367171+0000","last_fullsized":"2024-04-05T03:09:39.367171+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:51:09.230432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":117,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370062+0000","last_change":"2024-04-05T03:01:16.081559+0000","last_active":"2024-04-05T03:09:39.370062+0000","last_peered":"2024-04-05T03:09:39.370062+0000","last_clean":"2024-04-05T03:09:39.370062+0000","last_became_active":"2024-04-05T03:01:16.077624+0000","last_became_peered":"2024-04-05T03:01:16.077624+0000","last_unstale":"2024-04-05T03:09:39.370062+0000","last_undegraded":"2024-04-05T03:09:39.370062+0000","last_fullsized":"2024-04-05T03:09:39.370062+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:23:01.598037+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":69,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.376691+0000","last_change":"2024-04-05T03:09:39.376691+0000","last_active":"2024-04-05T03:09:39.376691+0000","last_peered":"2024-04-05T03:09:39.376691+0000","last_clean":"2024-04-05T03:09:39.376691+0000","last_became_active":"2024-04-05T03:09:39.376282+0000","last_became_peered":"2024-04-05T03:09:39.376282+0000","last_unstale":"2024-04-05T03:09:39.376691+0000","last_undegraded":"2024-04-05T03:09:39.376691+0000","last_fullsized":"2024-04-05T03:09:39.376691+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:29:43.628414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":106,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.365888+0000","last_change":"2024-04-05T03:04:44.542358+0000","last_active":"2024-04-05T03:09:39.365888+0000","last_peered":"2024-04-05T03:09:39.365888+0000","last_clean":"2024-04-05T03:09:39.365888+0000","last_became_active":"2024-04-05T03:04:44.540685+0000","last_became_peered":"2024-04-05T03:04:44.540685+0000","last_unstale":"2024-04-05T03:09:39.365888+0000","last_undegraded":"2024-04-05T03:09:39.365888+0000","last_fullsized":"2024-04-05T03:09:39.365888+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T04:06:49.360769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":73,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371105+0000","last_change":"2024-04-05T03:01:17.081782+0000","last_active":"2024-04-05T03:09:38.371105+0000","last_peered":"2024-04-05T03:09:38.371105+0000","last_clean":"2024-04-05T03:09:38.371105+0000","last_became_active":"2024-04-05T03:01:17.080953+0000","last_became_peered":"2024-04-05T03:01:17.080953+0000","last_unstale":"2024-04-05T03:09:38.371105+0000","last_undegraded":"2024-04-05T03:09:38.371105+0000","last_fullsized":"2024-04-05T03:09:38.371105+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:47:39.605395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":104,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.378269+0000","last_change":"2024-04-05T03:09:04.081811+0000","last_active":"2024-04-05T03:09:38.378269+0000","last_peered":"2024-04-05T03:09:38.378269+0000","last_clean":"2024-04-05T03:09:38.378269+0000","last_became_active":"2024-04-05T03:09:04.081494+0000","last_became_peered":"2024-04-05T03:09:04.081494+0000","last_unstale":"2024-04-05T03:09:38.378269+0000","last_undegraded":"2024-04-05T03:09:38.378269+0000","last_fullsized":"2024-04-05T03:09:38.378269+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:48:34.824572+0000","last_clean_scrub_stamp":"2024-04-05T02:48:34.824572+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:31:27.400390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":129,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370940+0000","last_change":"2024-04-05T03:04:44.544595+0000","last_active":"2024-04-05T03:09:39.370940+0000","last_peered":"2024-04-05T03:09:39.370940+0000","last_clean":"2024-04-05T03:09:39.370940+0000","last_became_active":"2024-04-05T03:04:44.543697+0000","last_became_peered":"2024-04-05T03:04:44.543697+0000","last_unstale":"2024-04-05T03:09:39.370940+0000","last_undegraded":"2024-04-05T03:09:39.370940+0000","last_fullsized":"2024-04-05T03:09:39.370940+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:09:19.091552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":103,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370969+0000","last_change":"2024-04-05T03:01:17.081314+0000","last_active":"2024-04-05T03:09:39.370969+0000","last_peered":"2024-04-05T03:09:39.370969+0000","last_clean":"2024-04-05T03:09:39.370969+0000","last_became_active":"2024-04-05T03:01:17.080933+0000","last_became_peered":"2024-04-05T03:01:17.080933+0000","last_unstale":"2024-04-05T03:09:39.370969+0000","last_undegraded":"2024-04-05T03:09:39.370969+0000","last_fullsized":"2024-04-05T03:09:39.370969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T07:45:23.984197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":119,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370448+0000","last_change":"2024-04-05T03:01:16.075975+0000","last_active":"2024-04-05T03:09:39.370448+0000","last_peered":"2024-04-05T03:09:39.370448+0000","last_clean":"2024-04-05T03:09:39.370448+0000","last_became_active":"2024-04-05T03:01:16.075414+0000","last_became_peered":"2024-04-05T03:01:16.075414+0000","last_unstale":"2024-04-05T03:09:39.370448+0000","last_undegraded":"2024-04-05T03:09:39.370448+0000","last_fullsized":"2024-04-05T03:09:39.370448+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T14:27:38.222255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.362574+0000","last_change":"2024-04-05T03:09:04.083841+0000","last_active":"2024-04-05T03:09:39.362574+0000","last_peered":"2024-04-05T03:09:39.362574+0000","last_clean":"2024-04-05T03:09:39.362574+0000","last_became_active":"2024-04-05T03:09:04.083518+0000","last_became_peered":"2024-04-05T03:09:04.083518+0000","last_unstale":"2024-04-05T03:09:39.362574+0000","last_undegraded":"2024-04-05T03:09:39.362574+0000","last_fullsized":"2024-04-05T03:09:39.362574+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T13:43:45.171084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.371831+0000","last_change":"2024-04-05T03:04:44.545489+0000","last_active":"2024-04-05T03:09:38.371831+0000","last_peered":"2024-04-05T03:09:38.371831+0000","last_clean":"2024-04-05T03:09:38.371831+0000","last_became_active":"2024-04-05T03:04:44.545029+0000","last_became_peered":"2024-04-05T03:04:44.545029+0000","last_unstale":"2024-04-05T03:09:38.371831+0000","last_undegraded":"2024-04-05T03:09:38.371831+0000","last_fullsized":"2024-04-05T03:09:38.371831+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T05:46:39.553004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.377407+0000","last_change":"2024-04-05T03:04:44.545251+0000","last_active":"2024-04-05T03:09:38.377407+0000","last_peered":"2024-04-05T03:09:38.377407+0000","last_clean":"2024-04-05T03:09:38.377407+0000","last_became_active":"2024-04-05T03:04:44.544613+0000","last_became_peered":"2024-04-05T03:04:44.544613+0000","last_unstale":"2024-04-05T03:09:38.377407+0000","last_undegraded":"2024-04-05T03:09:38.377407+0000","last_fullsized":"2024-04-05T03:09:38.377407+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T03:44:58.655232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":84,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.372376+0000","last_change":"2024-04-05T02:49:47.188962+0000","last_active":"2024-04-05T03:09:38.372376+0000","last_peered":"2024-04-05T03:09:38.372376+0000","last_clean":"2024-04-05T03:09:38.372376+0000","last_became_active":"2024-04-05T02:49:47.188681+0000","last_became_peered":"2024-04-05T02:49:47.188681+0000","last_unstale":"2024-04-05T03:09:38.372376+0000","last_undegraded":"2024-04-05T03:09:38.372376+0000","last_fullsized":"2024-04-05T03:09:38.372376+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:59:25.089268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.375048+0000","last_change":"2024-04-05T03:09:39.375048+0000","last_active":"2024-04-05T03:09:39.375048+0000","last_peered":"2024-04-05T03:09:39.375048+0000","last_clean":"2024-04-05T03:09:39.375048+0000","last_became_active":"2024-04-05T03:09:39.374676+0000","last_became_peered":"2024-04-05T03:09:39.374676+0000","last_unstale":"2024-04-05T03:09:39.375048+0000","last_undegraded":"2024-04-05T03:09:39.375048+0000","last_fullsized":"2024-04-05T03:09:39.375048+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:42:49.965978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":107,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.373194+0000","last_change":"2024-04-05T03:09:39.373194+0000","last_active":"2024-04-05T03:09:39.373194+0000","last_peered":"2024-04-05T03:09:39.373194+0000","last_clean":"2024-04-05T03:09:39.373194+0000","last_became_active":"2024-04-05T03:09:39.372713+0000","last_became_peered":"2024-04-05T03:09:39.372713+0000","last_unstale":"2024-04-05T03:09:39.373194+0000","last_undegraded":"2024-04-05T03:09:39.373194+0000","last_fullsized":"2024-04-05T03:09:39.373194+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T08:08:24.474831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.376476+0000","last_change":"2024-04-05T03:09:39.376476+0000","last_active":"2024-04-05T03:09:39.376476+0000","last_peered":"2024-04-05T03:09:39.376476+0000","last_clean":"2024-04-05T03:09:39.376476+0000","last_became_active":"2024-04-05T03:09:39.376084+0000","last_became_peered":"2024-04-05T03:09:39.376084+0000","last_unstale":"2024-04-05T03:09:39.376476+0000","last_undegraded":"2024-04-05T03:09:39.376476+0000","last_fullsized":"2024-04-05T03:09:39.376476+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T09:48:16.648435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":129,"reported_epoch":140,"state":"active+clean","last_fresh":"2024-04-05T03:09:39.370618+0000","last_change":"2024-04-05T03:09:04.082832+0000","last_active":"2024-04-05T03:09:39.370618+0000","last_peered":"2024-04-05T03:09:39.370618+0000","last_clean":"2024-04-05T03:09:39.370618+0000","last_became_active":"2024-04-05T03:09:04.082444+0000","last_became_peered":"2024-04-05T03:09:04.082444+0000","last_unstale":"2024-04-05T03:09:39.370618+0000","last_undegraded":"2024-04-05T03:09:39.370618+0000","last_fullsized":"2024-04-05T03:09:39.370618+0000","mapping_epoch":134,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":135,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:46.170655+0000","last_clean_scrub_stamp":"2024-04-05T02:49:46.170655+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T12:20:04.975531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2024-04-05T03:09:38.372573+0000","last_change":"2024-04-05T03:04:44.540054+0000","last_active":"2024-04-05T03:09:38.372573+0000","last_peered":"2024-04-05T03:09:38.372573+0000","last_clean":"2024-04-05T03:09:38.372573+0000","last_became_active":"2024-04-05T03:04:44.539734+0000","last_became_peered":"2024-04-05T03:04:44.539734+0000","last_unstale":"2024-04-05T03:09:38.372573+0000","last_undegraded":"2024-04-05T03:09:38.372573+0000","last_fullsized":"2024-04-05T03:09:38.372573+0000","mapping_epoch":119,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-05T02:49:47.172709+0000","last_clean_scrub_stamp":"2024-04-05T02:49:47.172709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-06T10:32:59.561866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13599,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201554,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":311728,"kb_used_data":2084,"kb_used_omap":1,"kb_used_meta":309630,"kb_avail":93433424,"statfs":{"total":95995035648,"available":95675826176,"internally_reserved":0,"allocated":2134016,"data_stored":1476078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317061464},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463192,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":299428,"kb_used_data":2072,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93445724,"statfs":{"total":95995035648,"available":95688421376,"internally_reserved":0,"allocated":2121728,"data_stored":1475932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":119,"seq":511101108297,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":229228,"kb_used_data":1560,"kb_used_omap":5,"kb_used_meta":227642,"kb_avail":93515924,"statfs":{"total":95995035648,"available":95760306176,"internally_reserved":0,"allocated":1597440,"data_stored":893673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5865,"internal_metadata":233105687},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":94,"seq":403726925938,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":248740,"kb_used_data":2192,"kb_used_omap":5,"kb_used_meta":246522,"kb_avail":93496412,"statfs":{"total":95995035648,"available":95740325888,"internally_reserved":0,"allocated":2244608,"data_stored":1487160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6054,"internal_metadata":252438618},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":134,"seq":575525617685,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":52872,"kb_used_data":1524,"kb_used_omap":6,"kb_used_meta":51321,"kb_avail":93692280,"statfs":{"total":95995035648,"available":95940894720,"internally_reserved":0,"allocated":1560576,"data_stored":888902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6668,"internal_metadata":52553204},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":139,"seq":597000454158,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":78376,"kb_used_data":1492,"kb_used_omap":4,"kb_used_meta":76859,"kb_avail":93666776,"statfs":{"total":95995035648,"available":95914778624,"internally_reserved":0,"allocated":1527808,"data_stored":884112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4639,"internal_metadata":78704097},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986465,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587240,"kb_used_data":1500,"kb_used_omap":1,"kb_used_meta":585726,"kb_avail":93157912,"statfs":{"total":95995035648,"available":95393701888,"internally_reserved":0,"allocated":1536000,"data_stored":885640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":599783768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724827,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":599512,"kb_used_data":1476,"kb_used_omap":1,"kb_used_meta":598014,"kb_avail":93145640,"statfs":{"total":95995035648,"available":95381135360,"internally_reserved":0,"allocated":1511424,"data_stored":880802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":612366680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4350,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4161,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2153,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2935,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-05T03:10:41.579 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-05T03:10:41.580 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-05T03:10:41.580 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-05T03:10:41.580 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:10:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:41.836 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:41.958 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:10:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:10:41] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.43.0" 2024-04-05T03:10:42.552 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:42 smithi150 ceph-mon[103445]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:10:42.552 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:42 smithi150 ceph-mon[103445]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:42.552 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:42 smithi150 ceph-mon[107701]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:10:42.552 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:42 smithi150 ceph-mon[107701]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:42.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:42 smithi196 ceph-mon[91352]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-05T03:10:42.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:42 smithi196 ceph-mon[91352]: pgmap v525: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:43.784 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:10:43.784 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:10:44.388 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:44 smithi150 ceph-mon[107701]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:44.388 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:44 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3745837783' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:44.389 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:44 smithi150 ceph-mon[103445]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:44.389 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:44 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3745837783' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:44.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:44 smithi196 ceph-mon[91352]: pgmap v526: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:44.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:44 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3745837783' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:46.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:46 smithi196 ceph-mon[91352]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:46.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:46 smithi150 ceph-mon[103445]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:46.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:46 smithi150 ceph-mon[107701]: pgmap v527: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:47.415 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:10:47.684 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:48 smithi150 ceph-mon[103445]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:48 smithi150 ceph-mon[107701]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:48.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:48 smithi196 ceph-mon[91352]: pgmap v528: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:49.553 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:10:49.553 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:10:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:50 smithi150 ceph-mon[107701]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:50.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:50 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/252443249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:50 smithi150 ceph-mon[103445]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:50.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:50 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/252443249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:50 smithi196 ceph-mon[91352]: pgmap v529: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:50.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:50 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/252443249' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:51.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:10:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:10:51] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.43.0" 2024-04-05T03:10:52.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:52 smithi196 ceph-mon[91352]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:52.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:52 smithi150 ceph-mon[107701]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:52 smithi150 ceph-mon[103445]: pgmap v530: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:53.146 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:10:53.424 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:10:54.517 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:54 smithi150 ceph-mon[107701]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:54.518 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:54 smithi150 ceph-mon[103445]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:54.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:54 smithi196 ceph-mon[91352]: pgmap v531: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:55.272 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:10:55.272 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:10:55.583 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:55 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:55.584 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:55 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:55.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:55 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:10:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:56 smithi196 ceph-mon[91352]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:56 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/678005812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:56 smithi150 ceph-mon[103445]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:56 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/678005812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:56 smithi150 ceph-mon[107701]: pgmap v532: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:56 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/678005812' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:10:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:10:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:10:58 smithi196 ceph-mon[91352]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:58.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:10:58 smithi150 ceph-mon[103445]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:58.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:10:58 smithi150 ceph-mon[107701]: pgmap v533: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:10:58.876 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:10:59.170 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:00.332 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:00.332 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.333 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:00.334 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:00.334 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:00 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: pgmap v534: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:11:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:00 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:11:01.083 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:01.084 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:01.381 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-mon[103445]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.381 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-mon[103445]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.381 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2152035571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:01.381 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-mon[107701]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.381 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-mon[107701]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.381 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2152035571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:01.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:01 smithi196 ceph-mon[91352]: pgmap v535: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:01 smithi196 ceph-mon[91352]: pgmap v536: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:01.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:01 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2152035571' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:01.681 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:11:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:11:01] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:11:03.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:03 smithi196 ceph-mon[91352]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:03.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:03 smithi150 ceph-mon[103445]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:03.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:03 smithi150 ceph-mon[107701]: pgmap v537: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:04.708 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:04.974 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:05.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:05 smithi150 ceph-mon[103445]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:05.350 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:05 smithi150 ceph-mon[107701]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:05.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:05 smithi196 ceph-mon[91352]: pgmap v538: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:06.942 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:06.942 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:07.537 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:07 smithi150 ceph-mon[103445]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:07.538 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:07 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/542156389' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:07.538 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:07 smithi150 ceph-mon[107701]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:07.538 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:07 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/542156389' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:07 smithi196 ceph-mon[91352]: pgmap v539: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:07.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:07 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/542156389' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:09.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:09 smithi196 ceph-mon[91352]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:09 smithi150 ceph-mon[103445]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:09.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:09 smithi150 ceph-mon[107701]: pgmap v540: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:10.562 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:10.833 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:11.521 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:11 smithi150 ceph-mon[103445]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:11.522 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:11.522 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:11 smithi150 ceph-mon[107701]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:11.522 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:11 smithi196 ceph-mon[91352]: pgmap v541: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:11.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:11.902 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:11:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:11:11] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:11:12.691 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:12.691 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:13.567 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:13 smithi150 ceph-mon[103445]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:13.567 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:13 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3595500887' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:13.567 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:13 smithi150 ceph-mon[107701]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:13.567 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:13 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3595500887' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:13.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:13 smithi196 ceph-mon[91352]: pgmap v542: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:13.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:13 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3595500887' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:15.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:15 smithi196 ceph-mon[91352]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:15 smithi150 ceph-mon[103445]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:15 smithi150 ceph-mon[107701]: pgmap v543: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:16.310 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:16.582 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:17.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:17 smithi196 ceph-mon[91352]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:17.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:17 smithi150 ceph-mon[103445]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:17.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:17 smithi150 ceph-mon[107701]: pgmap v544: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:18.631 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:18.631 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:19.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:19 smithi150 ceph-mon[103445]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:19.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:19 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2614671766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:19.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:19 smithi150 ceph-mon[107701]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:19.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:19 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2614671766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:19 smithi196 ceph-mon[91352]: pgmap v545: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:19 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2614671766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:21 smithi196 ceph-mon[91352]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:21.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:21 smithi150 ceph-mon[103445]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:21.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:21 smithi150 ceph-mon[107701]: pgmap v546: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:11:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:11:21] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:11:22.224 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:22.504 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:23.565 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:23 smithi150 ceph-mon[103445]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:23.566 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:23 smithi150 ceph-mon[107701]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:23.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:23 smithi196 ceph-mon[91352]: pgmap v547: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:24.647 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:24.647 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:25 smithi196 ceph-mon[91352]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:25.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:25 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/341481868' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:25 smithi150 ceph-mon[103445]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:25.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:25 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/341481868' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:25 smithi150 ceph-mon[107701]: pgmap v548: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:25.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:25 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/341481868' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:26.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:27.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:27 smithi196 ceph-mon[91352]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:27.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:27 smithi150 ceph-mon[107701]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:27.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:27 smithi150 ceph-mon[103445]: pgmap v549: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:28.282 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:28.552 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:29.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:29 smithi196 ceph-mon[91352]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:29.679 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:29 smithi150 ceph-mon[103445]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:29.679 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:29 smithi150 ceph-mon[107701]: pgmap v550: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:30.408 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:30.408 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:31 smithi196 ceph-mon[91352]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:31.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:31 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2860831887' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:31 smithi150 ceph-mon[103445]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:31.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:31 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2860831887' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:31.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:11:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:11:31] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:11:31.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:31 smithi150 ceph-mon[107701]: pgmap v551: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:31.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:31 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2860831887' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:33.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:33 smithi196 ceph-mon[91352]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:33.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:33 smithi150 ceph-mon[103445]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:33.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:33 smithi150 ceph-mon[107701]: pgmap v552: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:34.012 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:34.282 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:35.435 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:35 smithi150 ceph-mon[107701]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:35.435 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:35 smithi150 ceph-mon[103445]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:35.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:35 smithi196 ceph-mon[91352]: pgmap v553: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:36.230 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:36.230 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:36.551 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:36 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/77280244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:36.551 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:36 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/77280244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:36.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:36 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/77280244' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:37.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:37 smithi196 ceph-mon[91352]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:37.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:37 smithi150 ceph-mon[103445]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:37.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:37 smithi150 ceph-mon[107701]: pgmap v554: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:39.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:39 smithi196 ceph-mon[91352]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:39.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:39 smithi150 ceph-mon[103445]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:39 smithi150 ceph-mon[107701]: pgmap v555: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:39.878 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:40.145 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:41 smithi150 ceph-mon[103445]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:41.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:41 smithi150 ceph-mon[107701]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:41.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:41 smithi196 ceph-mon[91352]: pgmap v556: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:41.976 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:11:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:11:41] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:11:42.016 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:42.016 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:42.583 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:42 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2082930231' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:42.583 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:42 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2082930231' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:42.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:42 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2082930231' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:43.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:43 smithi196 ceph-mon[91352]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:43.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:43 smithi150 ceph-mon[103445]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:43.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:43 smithi150 ceph-mon[107701]: pgmap v557: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:45.609 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:45.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:45 smithi196 ceph-mon[91352]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:45.641 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:45 smithi150 ceph-mon[103445]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:45.641 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:45 smithi150 ceph-mon[107701]: pgmap v558: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:45.881 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:47.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:47 smithi196 ceph-mon[91352]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:47.709 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:47.709 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:47.710 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:47 smithi150 ceph-mon[103445]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:47.710 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:47 smithi150 ceph-mon[107701]: pgmap v559: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:48.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:48 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1997826317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:48.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:48 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1997826317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:48.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:48 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1997826317' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:49.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:49 smithi196 ceph-mon[91352]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:49.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:49 smithi150 ceph-mon[103445]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:49.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:49 smithi150 ceph-mon[107701]: pgmap v560: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:51.298 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:51.568 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:51.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:51 smithi196 ceph-mon[91352]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:51.689 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:51 smithi150 ceph-mon[103445]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:51.690 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:51 smithi150 ceph-mon[107701]: pgmap v561: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:51.690 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:11:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:11:51] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:11:53.571 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:53.571 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:53.571 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:53 smithi150 ceph-mon[103445]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:53.572 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:53 smithi150 ceph-mon[107701]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:53.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:53 smithi196 ceph-mon[91352]: pgmap v562: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:54.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:54 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2679197383' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:54.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:54 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2679197383' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:54 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2679197383' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:55.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:55 smithi150 ceph-mon[103445]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:55.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:55 smithi150 ceph-mon[107701]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:55.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:55 smithi196 ceph-mon[91352]: pgmap v563: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:56.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:56.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:11:57.213 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:11:57.484 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:11:57.607 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:57 smithi150 ceph-mon[103445]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:57.607 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:57 smithi150 ceph-mon[107701]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:57.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:57 smithi196 ceph-mon[91352]: pgmap v564: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:59.341 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:11:59.341 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:11:59.717 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:59 smithi150 ceph-mon[107701]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:59.718 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:11:59 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/182419361' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:59.718 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:59 smithi150 ceph-mon[103445]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:59.718 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:11:59 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/182419361' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:11:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:59 smithi196 ceph-mon[91352]: pgmap v565: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:11:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:11:59 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/182419361' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:00.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:00 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:00.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:00 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:00.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:00 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:12:01.700 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:01 smithi196 ceph-mon[91352]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:01.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:01 smithi150 ceph-mon[103445]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:01.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:01 smithi150 ceph-mon[107701]: pgmap v566: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:01.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:12:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:12:01] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:12:03.027 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:03.665 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:03 smithi196 ceph-mon[91352]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:03.705 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:03 smithi150 ceph-mon[103445]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:03.706 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:03 smithi150 ceph-mon[107701]: pgmap v567: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:03.881 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:05.015 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.016 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:05.017 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.017 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.017 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:05.017 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:05.017 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:04 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: pgmap v568: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:12:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:04 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:12:05.799 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:05.800 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:06.100 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:05 smithi150 ceph-mon[103445]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.100 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:05 smithi150 ceph-mon[103445]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.100 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2221688088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:06.100 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:05 smithi150 ceph-mon[107701]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.101 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:05 smithi150 ceph-mon[107701]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.101 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2221688088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:05 smithi196 ceph-mon[91352]: pgmap v569: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:05 smithi196 ceph-mon[91352]: pgmap v570: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2221688088' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:08.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:07 smithi150 ceph-mon[103445]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:08.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:07 smithi150 ceph-mon[107701]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:08.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:07 smithi196 ceph-mon[91352]: pgmap v571: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:09.401 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:09.669 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:10.069 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:09 smithi150 ceph-mon[103445]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:10.070 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:09 smithi150 ceph-mon[107701]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:10.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:09 smithi196 ceph-mon[91352]: pgmap v572: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:11.137 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:10 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:11.138 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:10 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:11.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:10 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:11.631 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:11.631 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:11.931 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:12:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:12:11] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-04-05T03:12:12.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:11 smithi150 ceph-mon[107701]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:12.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:11 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1185481462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:12.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:11 smithi150 ceph-mon[103445]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:12.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:11 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1185481462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:11 smithi196 ceph-mon[91352]: pgmap v573: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:12.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:11 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1185481462' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:14.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:13 smithi196 ceph-mon[91352]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:14.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:13 smithi150 ceph-mon[103445]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:14.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:13 smithi150 ceph-mon[107701]: pgmap v574: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:15.274 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:15.548 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:16.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:15 smithi150 ceph-mon[103445]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:16.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:15 smithi150 ceph-mon[107701]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:16.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:15 smithi196 ceph-mon[91352]: pgmap v575: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:17.638 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:17.639 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:17 smithi150 ceph-mon[103445]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:17 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1383659596' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:17 smithi150 ceph-mon[107701]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:18.221 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:17 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1383659596' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:18.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:17 smithi196 ceph-mon[91352]: pgmap v576: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:18.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:17 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1383659596' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:20.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:19 smithi150 ceph-mon[103445]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:20.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:19 smithi150 ceph-mon[107701]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:20.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:19 smithi196 ceph-mon[91352]: pgmap v577: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:21.246 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:21.515 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:21.637 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:12:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:12:21] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-04-05T03:12:22.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:21 smithi150 ceph-mon[103445]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:22.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:21 smithi150 ceph-mon[107701]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:22.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:21 smithi196 ceph-mon[91352]: pgmap v578: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:23.534 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:23.534 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:24.086 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:23 smithi150 ceph-mon[107701]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:24.086 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:23 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2881167799' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:24.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:23 smithi150 ceph-mon[103445]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:24.087 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:23 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2881167799' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:23 smithi196 ceph-mon[91352]: pgmap v579: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:24.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:23 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2881167799' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:25 smithi150 ceph-mon[107701]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:26.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:25 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:25 smithi150 ceph-mon[103445]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:26.228 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:25 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:25 smithi196 ceph-mon[91352]: pgmap v580: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:26.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:25 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:27.111 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:27.382 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:28.226 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:27 smithi150 ceph-mon[103445]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:28.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:27 smithi150 ceph-mon[107701]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:28.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:27 smithi196 ceph-mon[91352]: pgmap v581: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:29.315 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:29.316 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:29 smithi150 ceph-mon[103445]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:30.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1162370368' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:30.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:29 smithi150 ceph-mon[107701]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:30.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1162370368' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:30.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:29 smithi196 ceph-mon[91352]: pgmap v582: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:30.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1162370368' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:31.976 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:12:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:12:31] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:12:32.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:31 smithi196 ceph-mon[91352]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:32.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:31 smithi150 ceph-mon[103445]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:32.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:31 smithi150 ceph-mon[107701]: pgmap v583: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:32.901 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:33.171 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:34.290 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:33 smithi150 ceph-mon[103445]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:34.291 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:33 smithi150 ceph-mon[107701]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:34.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:33 smithi196 ceph-mon[91352]: pgmap v584: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:35.071 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:35.071 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:35 smithi196 ceph-mon[91352]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:36.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2084612753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:36.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:35 smithi150 ceph-mon[107701]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:36.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2084612753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:36.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:35 smithi150 ceph-mon[103445]: pgmap v585: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:36.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2084612753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:38.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:37 smithi196 ceph-mon[91352]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:38.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:37 smithi150 ceph-mon[107701]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:38.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:37 smithi150 ceph-mon[103445]: pgmap v586: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:38.698 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:38.967 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:40.063 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:39 smithi150 ceph-mon[107701]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:40.063 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:39 smithi150 ceph-mon[103445]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:40.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:39 smithi196 ceph-mon[91352]: pgmap v587: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:40.820 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:40.821 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:41.186 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:41.187 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2745980215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:41.187 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:41.187 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2745980215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:41.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2745980215' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:41.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:12:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:12:41] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2024-04-05T03:12:42.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:42 smithi196 ceph-mon[91352]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:42.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:42 smithi150 ceph-mon[103445]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:42.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:42 smithi150 ceph-mon[107701]: pgmap v588: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:44.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:44 smithi196 ceph-mon[91352]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:44.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:44 smithi150 ceph-mon[103445]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:44.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:44 smithi150 ceph-mon[107701]: pgmap v589: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:44.500 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:44.775 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:46.201 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:46 smithi150 ceph-mon[103445]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:46.201 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:46 smithi150 ceph-mon[107701]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:46.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:46 smithi196 ceph-mon[91352]: pgmap v590: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:46.677 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:46.677 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:47.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:47 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/780678059' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:47.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:47 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/780678059' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:47.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:47 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/780678059' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:48.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:48 smithi196 ceph-mon[91352]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:48.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:48 smithi150 ceph-mon[103445]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:48.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:48 smithi150 ceph-mon[107701]: pgmap v591: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:50.293 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:50.324 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:50 smithi150 ceph-mon[103445]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:50.324 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:50 smithi150 ceph-mon[107701]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:50.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:50 smithi196 ceph-mon[91352]: pgmap v592: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:50.561 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:51.706 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:12:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:12:51] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2024-04-05T03:12:52.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:52 smithi196 ceph-mon[91352]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:52.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:52 smithi150 ceph-mon[103445]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:52.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:52 smithi150 ceph-mon[107701]: pgmap v593: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:52.477 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:52.477 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:53.139 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:53 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2766623696' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:53.139 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:53 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2766623696' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:53.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:53 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2766623696' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:54.226 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:54 smithi150 ceph-mon[103445]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:54.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:54 smithi150 ceph-mon[107701]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:54.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:54 smithi196 ceph-mon[91352]: pgmap v594: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:56.164 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:12:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:56 smithi196 ceph-mon[91352]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:56.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:56.434 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:12:56.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:56 smithi150 ceph-mon[103445]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:56.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:56 smithi150 ceph-mon[107701]: pgmap v595: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:56.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:12:58.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:58 smithi150 ceph-mon[103445]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:58.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:58 smithi150 ceph-mon[107701]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:58.362 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:12:58.362 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:12:58.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:58 smithi196 ceph-mon[91352]: pgmap v596: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:12:59.226 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:12:59 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/605204610' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:59.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:12:59 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/605204610' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:12:59.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:12:59 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/605204610' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:00.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:00 smithi196 ceph-mon[91352]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:00.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:00 smithi150 ceph-mon[103445]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:00.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:00 smithi150 ceph-mon[107701]: pgmap v597: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:01.972 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:01.976 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:13:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:13:01] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:13:02.241 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:02.363 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:02 smithi150 ceph-mon[103445]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:02.363 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:02 smithi150 ceph-mon[107701]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:02.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:02 smithi196 ceph-mon[91352]: pgmap v598: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:04.201 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:04.201 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:04.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:04 smithi196 ceph-mon[91352]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:04.476 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:04 smithi150 ceph-mon[107701]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:04.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:04 smithi150 ceph-mon[103445]: pgmap v599: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:05.385 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:05 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3693782478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:05.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:05 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:05.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:05 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3693782478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:05.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:05 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:05.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:05 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3693782478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:05.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:05 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:13:06.386 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:06 smithi196 ceph-mon[91352]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:06.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:06 smithi150 ceph-mon[103445]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:06.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:06 smithi150 ceph-mon[107701]: pgmap v600: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:07.812 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:08.140 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:08 smithi150 ceph-mon[103445]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:08.140 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:08 smithi150 ceph-mon[107701]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:08.309 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:08 smithi196 ceph-mon[91352]: pgmap v601: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:08.598 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:09.833 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.834 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:09.835 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:09.835 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:09.835 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:09 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:13:10.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:09 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:13:10.565 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:10.565 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[107701]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[107701]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[107701]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2415329677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[103445]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[103445]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[103445]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:10.865 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2415329677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:10 smithi196 ceph-mon[91352]: pgmap v602: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:10 smithi196 ceph-mon[91352]: pgmap v603: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:10 smithi196 ceph-mon[91352]: pgmap v604: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:11.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2415329677' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:11.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:11.978 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:13:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:13:11] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:13:11.978 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:12.135 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:12.756 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:12 smithi150 ceph-mon[103445]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:12.757 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:12 smithi150 ceph-mon[107701]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:13.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:12 smithi196 ceph-mon[91352]: pgmap v605: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:14.151 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:14.410 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:14.810 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:14 smithi150 ceph-mon[103445]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:14.810 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:14 smithi150 ceph-mon[107701]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:15.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:14 smithi196 ceph-mon[91352]: pgmap v606: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:16.298 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:16.298 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:16.887 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:16 smithi150 ceph-mon[103445]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:16.887 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:16 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3348919613' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:16.887 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:16 smithi150 ceph-mon[107701]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:16.887 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:16 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3348919613' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:17.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:16 smithi196 ceph-mon[91352]: pgmap v607: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:17.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:16 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3348919613' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:18.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:18 smithi150 ceph-mon[103445]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:18.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:18 smithi150 ceph-mon[107701]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:19.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:18 smithi196 ceph-mon[91352]: pgmap v608: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:19.889 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:20.159 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:20.886 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:20 smithi150 ceph-mon[103445]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:20.886 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:20 smithi150 ceph-mon[107701]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:21.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:20 smithi196 ceph-mon[91352]: pgmap v609: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:21.726 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:13:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:13:21] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:13:21.895 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:21.895 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:22.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:22 smithi150 ceph-mon[107701]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:22.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:22 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/23549311' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:22 smithi150 ceph-mon[103445]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:22.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:22 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/23549311' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:23.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:22 smithi196 ceph-mon[91352]: pgmap v610: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:23.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:22 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/23549311' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:24.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:24 smithi150 ceph-mon[103445]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:24.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:24 smithi150 ceph-mon[107701]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:25.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:24 smithi196 ceph-mon[91352]: pgmap v611: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:25.563 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:25.837 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:26.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:26 smithi150 ceph-mon[107701]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:26.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:26 smithi150 ceph-mon[103445]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:26.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:27.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:26 smithi196 ceph-mon[91352]: pgmap v612: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:27.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:27.793 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:27.794 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:28.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:28 smithi150 ceph-mon[103445]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:28.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:28 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2054105038' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:28.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:28 smithi150 ceph-mon[107701]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:28.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:28 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2054105038' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:29.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:28 smithi196 ceph-mon[91352]: pgmap v613: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:29.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:28 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2054105038' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:30.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:30 smithi150 ceph-mon[103445]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:30.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:30 smithi150 ceph-mon[107701]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:31.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:30 smithi196 ceph-mon[91352]: pgmap v614: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:31.437 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:31.707 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:31.726 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:13:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:13:31] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.43.0" 2024-04-05T03:13:32.899 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:32 smithi150 ceph-mon[107701]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:32.900 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:32 smithi150 ceph-mon[103445]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:33.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:32 smithi196 ceph-mon[91352]: pgmap v615: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:33.843 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:33.843 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:34.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:34 smithi150 ceph-mon[103445]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:34.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:34 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/104382948' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:34 smithi150 ceph-mon[107701]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:34 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/104382948' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:35.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:34 smithi196 ceph-mon[91352]: pgmap v616: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:35.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:34 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/104382948' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:36.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:36 smithi150 ceph-mon[103445]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:36.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:36 smithi150 ceph-mon[107701]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:37.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:36 smithi196 ceph-mon[91352]: pgmap v617: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:37.568 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:37.837 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:38.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:38 smithi150 ceph-mon[103445]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:38.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:38 smithi150 ceph-mon[107701]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:39.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:38 smithi196 ceph-mon[91352]: pgmap v618: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:39.850 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:39.850 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:40.726 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:40 smithi150 ceph-mon[107701]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:40.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1847185666' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:40.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:40 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:40.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:40 smithi150 ceph-mon[103445]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:40.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1847185666' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:40.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:40 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:41.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:40 smithi196 ceph-mon[91352]: pgmap v619: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:41.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1847185666' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:41.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:40 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:41.904 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:13:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:13:41] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:13:42.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:42 smithi150 ceph-mon[103445]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:42.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:42 smithi150 ceph-mon[107701]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:43.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:42 smithi196 ceph-mon[91352]: pgmap v620: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:43.500 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:43.766 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:44.921 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:44 smithi150 ceph-mon[107701]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:44.922 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:44 smithi150 ceph-mon[103445]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:45.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:44 smithi196 ceph-mon[91352]: pgmap v621: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:45.690 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:45.691 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:45.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:45 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2156538399' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:45.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:45 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2156538399' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:46.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:45 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2156538399' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:47.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:46 smithi196 ceph-mon[91352]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:47.226 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:46 smithi150 ceph-mon[103445]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:47.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:46 smithi150 ceph-mon[107701]: pgmap v622: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:48.136 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:47 smithi196 ceph-mon[91352]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:47 smithi150 ceph-mon[103445]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:48.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:47 smithi150 ceph-mon[107701]: pgmap v623: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:49.424 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:49.697 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:50.878 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:50 smithi150 ceph-mon[103445]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:50.878 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:50 smithi150 ceph-mon[107701]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:50.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:50 smithi196 ceph-mon[91352]: pgmap v624: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:51.650 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:51.650 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:51.968 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:13:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:13:51] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:13:52.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:52 smithi150 ceph-mon[103445]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:52.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/16362259' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:52.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:52 smithi150 ceph-mon[107701]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:52.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/16362259' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:52.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:52 smithi196 ceph-mon[91352]: pgmap v625: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/16362259' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:54.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:54 smithi196 ceph-mon[91352]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:54.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:54 smithi150 ceph-mon[103445]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:54.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:54 smithi150 ceph-mon[107701]: pgmap v626: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:55.403 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:13:55.674 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:13:56.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:56 smithi196 ceph-mon[91352]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:56.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:56.917 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:56 smithi150 ceph-mon[103445]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:56.917 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:56.918 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:56 smithi150 ceph-mon[107701]: pgmap v627: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:56.918 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:13:57.684 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:13:57.685 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:13:58.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:58 smithi196 ceph-mon[91352]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:58.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:13:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3867395016' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:58.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:58 smithi150 ceph-mon[103445]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:58.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:13:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3867395016' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:13:58.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:58 smithi150 ceph-mon[107701]: pgmap v628: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:13:58.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:13:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3867395016' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:00.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:00 smithi196 ceph-mon[91352]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:00.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:00 smithi150 ceph-mon[103445]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:00.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:00 smithi150 ceph-mon[107701]: pgmap v629: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:01.325 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:01.595 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:01.717 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:14:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:14:01] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-04-05T03:14:02.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:02 smithi150 ceph-mon[103445]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:02.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:02 smithi150 ceph-mon[107701]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:02.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:02 smithi196 ceph-mon[91352]: pgmap v630: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:03.606 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:03.606 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:04.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:04 smithi196 ceph-mon[91352]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:04 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/178456120' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:04.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:04 smithi150 ceph-mon[107701]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:04.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:04 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/178456120' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:04 smithi150 ceph-mon[103445]: pgmap v631: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:04.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:04 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/178456120' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:06.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:06 smithi196 ceph-mon[91352]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:06.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:06 smithi150 ceph-mon[103445]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:06.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:06 smithi150 ceph-mon[107701]: pgmap v632: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:07.259 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:07.526 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:08.697 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:08 smithi150 ceph-mon[103445]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:08.697 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:08 smithi150 ceph-mon[107701]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:08.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:08 smithi196 ceph-mon[91352]: pgmap v633: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:09.471 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:09.471 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:09.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:09 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1029311991' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:09.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:09 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:14:09.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:09 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1029311991' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:09.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:09 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:14:09.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:09 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1029311991' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:09.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:09 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:14:10.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:10 smithi196 ceph-mon[91352]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:10.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:10 smithi150 ceph-mon[103445]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:10.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:10 smithi150 ceph-mon[107701]: pgmap v634: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:11.878 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:11.878 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:14:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:14:11] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:14:11.879 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:11.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:12.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:12 smithi196 ceph-mon[91352]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:12.894 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:12 smithi150 ceph-mon[107701]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:12.894 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:12 smithi150 ceph-mon[103445]: pgmap v635: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:13.097 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:13.401 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:14:14.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:14:14.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:14 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: pgmap v636: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:14:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:14 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:14:15.437 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:15.437 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:15.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:15 smithi150 ceph-mon[103445]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:15 smithi150 ceph-mon[103445]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:15 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3431632372' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:15 smithi150 ceph-mon[107701]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:15 smithi150 ceph-mon[107701]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:15 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3431632372' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:15.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:15 smithi196 ceph-mon[91352]: pgmap v637: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:15.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:15 smithi196 ceph-mon[91352]: pgmap v638: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:15.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:15 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3431632372' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:17.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:17 smithi150 ceph-mon[103445]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:17.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:17 smithi150 ceph-mon[107701]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:17.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:17 smithi196 ceph-mon[91352]: pgmap v639: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:19.096 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:19.365 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:19.739 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:19 smithi150 ceph-mon[103445]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:19.739 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:19 smithi150 ceph-mon[107701]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:19.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:19 smithi196 ceph-mon[91352]: pgmap v640: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:21.336 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:21.336 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:21.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:21 smithi150 ceph-mon[103445]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:21.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:21 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3447245145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:21.727 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:14:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:14:21] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2024-04-05T03:14:21.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:21 smithi150 ceph-mon[107701]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:21.728 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:21 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3447245145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:21.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:21 smithi196 ceph-mon[91352]: pgmap v641: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:21.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:21 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3447245145' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:23.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:23 smithi196 ceph-mon[91352]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:23.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:23 smithi150 ceph-mon[103445]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:23.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:23 smithi150 ceph-mon[107701]: pgmap v642: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:25.311 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:25.582 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:25.704 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:25 smithi150 ceph-mon[103445]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:25.704 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:25 smithi150 ceph-mon[107701]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:25.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:25 smithi196 ceph-mon[91352]: pgmap v643: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:26.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:26.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:26.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:27.528 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:27 smithi150 ceph-mon[107701]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:27.529 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:27 smithi150 ceph-mon[103445]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:27.768 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:27.768 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:27.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:27 smithi196 ceph-mon[91352]: pgmap v644: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:28.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:28 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3303198824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:28.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:28 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3303198824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:28.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:28 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3303198824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:29.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:29 smithi196 ceph-mon[91352]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:29.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:29 smithi150 ceph-mon[103445]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:29.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:29 smithi150 ceph-mon[107701]: pgmap v645: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:31.492 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:31.765 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:31.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:31 smithi196 ceph-mon[91352]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:31.886 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:31 smithi150 ceph-mon[103445]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:31.886 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:31 smithi150 ceph-mon[107701]: pgmap v646: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:31.887 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:14:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:14:31] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:14:33.610 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:33 smithi150 ceph-mon[103445]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:33.611 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:33 smithi150 ceph-mon[107701]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:33.876 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:33.877 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:33.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:33 smithi196 ceph-mon[91352]: pgmap v647: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:34.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:34 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3539814818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:34.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:34 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3539814818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:34.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:34 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3539814818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:35.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:35 smithi196 ceph-mon[91352]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:35.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:35 smithi150 ceph-mon[103445]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:35.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:35 smithi150 ceph-mon[107701]: pgmap v648: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:37.528 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:37.797 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:37.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:37 smithi196 ceph-mon[91352]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:37.918 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:37 smithi150 ceph-mon[103445]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:37.919 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:37 smithi150 ceph-mon[107701]: pgmap v649: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:39.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:39 smithi150 ceph-mon[103445]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:39.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:39 smithi150 ceph-mon[107701]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:39.789 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:39.790 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:39.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:39 smithi196 ceph-mon[91352]: pgmap v650: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:40.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:40 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3504657058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:40.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:40 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3504657058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:40.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:40 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3504657058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:41.878 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:41 smithi150 ceph-mon[103445]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:41.879 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:41.879 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:14:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:14:41] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-04-05T03:14:41.879 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:41 smithi150 ceph-mon[107701]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:41.879 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:41.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:41 smithi196 ceph-mon[91352]: pgmap v651: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:41.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:43.505 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:43.777 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:43.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:43 smithi196 ceph-mon[91352]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:43.900 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:43 smithi150 ceph-mon[103445]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:43.900 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:43 smithi150 ceph-mon[107701]: pgmap v652: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:45.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:45 smithi150 ceph-mon[103445]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:45.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:45 smithi150 ceph-mon[107701]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:45.781 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:45.781 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:45.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:45 smithi196 ceph-mon[91352]: pgmap v653: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:46.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:46 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2592313251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:46.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:46 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2592313251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:46.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:46 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2592313251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:47.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:47 smithi196 ceph-mon[91352]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:47.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:47 smithi150 ceph-mon[103445]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:47.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:47 smithi150 ceph-mon[107701]: pgmap v654: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:49.574 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:49.844 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:49.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:49 smithi196 ceph-mon[91352]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:49.966 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:49 smithi150 ceph-mon[103445]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:49.967 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:49 smithi150 ceph-mon[107701]: pgmap v655: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:51.672 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:51 smithi150 ceph-mon[103445]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:51.673 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:51 smithi150 ceph-mon[107701]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:51.673 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:14:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:14:51] "GET /metrics HTTP/1.1" 200 34117 "" "Prometheus/2.43.0" 2024-04-05T03:14:51.880 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:51.880 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:51.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:51 smithi196 ceph-mon[91352]: pgmap v656: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:52.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:52 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4271942392' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:52.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:52 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4271942392' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:52.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:52 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4271942392' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:53.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:53 smithi196 ceph-mon[91352]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:53.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:53 smithi150 ceph-mon[103445]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:53.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:53 smithi150 ceph-mon[107701]: pgmap v657: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:55.586 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:14:55.856 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:14:55.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:55 smithi196 ceph-mon[91352]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:55.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:55 smithi150 ceph-mon[103445]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:55.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:55 smithi150 ceph-mon[107701]: pgmap v658: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:56.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:56 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:56.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:56 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:56.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:56 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:14:57.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:57 smithi150 ceph-mon[103445]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:57.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:57 smithi150 ceph-mon[107701]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:57.868 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:14:57.868 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:14:57.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:57 smithi196 ceph-mon[91352]: pgmap v659: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:58.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:58 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2083927539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:58.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:58 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2083927539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:58.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:58 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2083927539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:14:59.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:14:59 smithi196 ceph-mon[91352]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:59.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:14:59 smithi150 ceph-mon[103445]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:14:59.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:14:59 smithi150 ceph-mon[107701]: pgmap v660: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:01.687 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:01.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:01 smithi196 ceph-mon[91352]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:01.956 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:01.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:01 smithi150 ceph-mon[103445]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:01.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:01 smithi150 ceph-mon[107701]: pgmap v661: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:01.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:15:01] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2024-04-05T03:15:03.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:03 smithi196 ceph-mon[91352]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:03.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:03 smithi150 ceph-mon[103445]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:03.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:03 smithi150 ceph-mon[107701]: pgmap v662: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:03.977 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:03.978 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:04.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:04 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/4168124558' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:04.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:04 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/4168124558' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:04.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:04 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/4168124558' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:05.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:05 smithi196 ceph-mon[91352]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:05.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:05 smithi150 ceph-mon[107701]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:05.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:05 smithi150 ceph-mon[103445]: pgmap v663: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:07.802 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:07.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:07 smithi196 ceph-mon[91352]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:07.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:07 smithi150 ceph-mon[103445]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:07.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:07 smithi150 ceph-mon[107701]: pgmap v664: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:08.070 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:09.625 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:09 smithi150 ceph-mon[107701]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:09.626 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:09 smithi150 ceph-mon[103445]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:09.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:09 smithi196 ceph-mon[91352]: pgmap v665: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:10.152 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:10.152 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:10.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:10 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1712743061' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:10.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:10 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1712743061' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:10.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:10 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1712743061' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:11 smithi196 ceph-mon[91352]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:11.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:11 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:11.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:11 smithi150 ceph-mon[103445]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:11.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:11 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:11.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:11 smithi150 ceph-mon[107701]: pgmap v666: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:11.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:11 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:11.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:11 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:15:11] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.43.0" 2024-04-05T03:15:13.726 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:13 smithi150 ceph-mon[103445]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:13.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:13 smithi150 ceph-mon[107701]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:13.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:13 smithi196 ceph-mon[91352]: pgmap v667: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:13.893 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:14.156 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:14.886 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:14 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:15:14.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:14 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:15:14.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:14 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-05T03:15:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:15 smithi150 ceph-mon[103445]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:15.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:15 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:15 smithi150 ceph-mon[107701]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:15 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:15.727 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:15 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:15.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:15 smithi196 ceph-mon[91352]: pgmap v668: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:15.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:15 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:15.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:15 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:16.251 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:16.251 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:16.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:16 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2826091234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:16.976 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:16 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2826091234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:16.977 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:16 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2826091234' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:17.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:17 smithi196 ceph-mon[91352]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:17.976 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:17 smithi150 ceph-mon[103445]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:17.977 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:17 smithi150 ceph-mon[107701]: pgmap v669: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:19.226 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:19 smithi150 ceph-mon[103445]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:19.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:19 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:19.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:19 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:19.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:19 smithi150 ceph-mon[107701]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:19.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:19 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:19.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:19 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:19.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:19 smithi196 ceph-mon[91352]: pgmap v670: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:19 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:19.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:19 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:20.170 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:20.446 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[103445]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[103445]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[107701]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:15:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:15:21.478 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-mon[107701]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:21 smithi196 ceph-mon[91352]: pgmap v671: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:21 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:21 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-05T03:15:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:21 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-05T03:15:21.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:21 smithi196 ceph-mon[91352]: from='mgr.25295 ' entity='mgr.y' 2024-04-05T03:15:21.792 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:21 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:15:21] "GET /metrics HTTP/1.1" 200 34105 "" "Prometheus/2.43.0" 2024-04-05T03:15:22.414 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:22.414 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:22.414 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:22 smithi150 ceph-mon[103445]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:22.414 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:22 smithi150 ceph-mon[103445]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:22.415 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:22 smithi150 ceph-mon[107701]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:22.415 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:22 smithi150 ceph-mon[107701]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:22 smithi196 ceph-mon[91352]: pgmap v672: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:22.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:22 smithi196 ceph-mon[91352]: pgmap v673: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:23.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:23 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1948960843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:23.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:23 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1948960843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:23.637 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:23 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1948960843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:24.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:24 smithi150 ceph-mon[103445]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:24.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:24 smithi150 ceph-mon[107701]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:24.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:24 smithi196 ceph-mon[91352]: pgmap v674: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:26.058 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:26.329 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:26.451 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:26 smithi150 ceph-mon[103445]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:26.451 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:26 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:26.451 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:26 smithi150 ceph-mon[107701]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:26.451 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:26 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:26.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:26 smithi196 ceph-mon[91352]: pgmap v675: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:26.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:26 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:28.382 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:28.382 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:28.476 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:28 smithi150 ceph-mon[107701]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:28.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:28 smithi150 ceph-mon[103445]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:28.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:28 smithi196 ceph-mon[91352]: pgmap v676: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:29.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:29 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/585873534' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:29.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:29 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/585873534' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:29.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:29 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/585873534' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:30.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:30 smithi150 ceph-mon[103445]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:30.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:30 smithi150 ceph-mon[107701]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:30.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:30 smithi196 ceph-mon[91352]: pgmap v677: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:31.977 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:31 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:15:31] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.43.0" 2024-04-05T03:15:32.150 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:32.419 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:32.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:32 smithi150 ceph-mon[103445]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:32.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:32 smithi150 ceph-mon[107701]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:32.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:32 smithi196 ceph-mon[91352]: pgmap v678: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:34.398 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:34.398 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:34.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:34 smithi150 ceph-mon[103445]: pgmap v679: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:34.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:34 smithi150 ceph-mon[107701]: pgmap v679: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:34.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:34 smithi196 ceph-mon[91352]: pgmap v679: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:35.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:35 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/2033435775' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:35.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:35 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/2033435775' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:35.633 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:35 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/2033435775' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:36.476 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:36 smithi150 ceph-mon[107701]: pgmap v680: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:36.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:36 smithi150 ceph-mon[103445]: pgmap v680: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:36.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:36 smithi196 ceph-mon[91352]: pgmap v680: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:38.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:38.360 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:38.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:38 smithi150 ceph-mon[103445]: pgmap v681: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:38.476 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:38 smithi150 ceph-mon[107701]: pgmap v681: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:38.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:38 smithi196 ceph-mon[91352]: pgmap v681: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:40.350 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:40 smithi150 ceph-mon[103445]: pgmap v682: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:40.351 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:40 smithi150 ceph-mon[107701]: pgmap v682: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:40.563 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:40.564 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:40.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:40 smithi196 ceph-mon[91352]: pgmap v682: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:41.521 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:41 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/1478798428' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:41.522 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:41 smithi150 ceph-mon[103445]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:41.522 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:41 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/1478798428' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:41.522 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:41 smithi150 ceph-mon[107701]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:41 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/1478798428' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:41.636 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:41 smithi196 ceph-mon[91352]: from='mgr.25295 172.21.15.150:0/746635080' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-05T03:15:41.976 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:41 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:15:41] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:15:42.599 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:42 smithi150 ceph-mon[103445]: pgmap v683: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:42.599 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:42 smithi150 ceph-mon[107701]: pgmap v683: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:42.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:42 smithi196 ceph-mon[91352]: pgmap v683: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:44.322 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 -- ceph health --format=json 2024-04-05T03:15:44.593 INFO:teuthology.orchestra.run.smithi150.stderr:Inferring config /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/mon.a/config 2024-04-05T03:15:44.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:44 smithi196 ceph-mon[91352]: pgmap v684: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:44.715 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:44 smithi150 ceph-mon[103445]: pgmap v684: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:44.716 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:44 smithi150 ceph-mon[107701]: pgmap v684: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:46.477 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:46 smithi150 ceph-mon[103445]: pgmap v685: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:46.477 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:46 smithi150 ceph-mon[107701]: pgmap v685: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:46.529 INFO:teuthology.orchestra.run.smithi150.stdout: 2024-04-05T03:15:46.529 INFO:teuthology.orchestra.run.smithi150.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-04-05T03:15:46.635 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:46 smithi196 ceph-mon[91352]: pgmap v685: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-04-05T03:15:47.212 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-05T03:15:47.441 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=35045d7b80ab47549d842577f8e5c177 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_9f192350ba5fe6d0a365b251a8c2af61f56a5fa6/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-05T03:15:47.444 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-05T03:15:47.453 INFO:tasks.cephadm:Teardown begin 2024-04-05T03:15:47.453 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:15:47.476 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:47 smithi150 ceph-mon[103445]: from='client.? 172.21.15.150:0/3870571837' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:47.476 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:47 smithi150 ceph-mon[107701]: from='client.? 172.21.15.150:0/3870571837' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:47.488 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:15:47.520 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:47 smithi196 ceph-mon[91352]: from='client.? 172.21.15.150:0/3870571837' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-05T03:15:47.525 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-05T03:15:47.526 DEBUG:teuthology.orchestra.run.smithi150:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-05T03:15:47.545 DEBUG:teuthology.orchestra.run.smithi196:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-05T03:15:47.583 INFO:tasks.cephadm:Stopping all daemons... 2024-04-05T03:15:47.583 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-05T03:15:47.583 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a 2024-04-05T03:15:47.893 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:47 smithi150 systemd[1]: Stopping Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:15:48.226 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[103420]: 2024-04-05T03:15:47.894+0000 7f5a31f38700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:15:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:47 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a[103420]: 2024-04-05T03:15:47.894+0000 7f5a31f38700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-05T03:15:48.227 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:48 smithi150 podman[165376]: 2024-04-05 03:15:48.138936177 +0000 UTC m=+0.368041152 container died 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-05T03:15:48.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:48 smithi150 podman[165376]: 2024-04-05 03:15:48.624885271 +0000 UTC m=+0.853990246 container cleanup 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-05T03:15:48.727 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:48 smithi150 bash[165376]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a 2024-04-05T03:15:49.091 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service' 2024-04-05T03:15:49.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:48 smithi150 podman[165395]: 2024-04-05 03:15:48.843200949 +0000 UTC m=+0.702999268 container remove 39df812ce0462613800bb10a3cea260d838248d803829339ab62dc8f86a0d49d (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-a, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, RELEASE=squid-0028525, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-04-05T03:15:49.115 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:49 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service: Deactivated successfully. 2024-04-05T03:15:49.116 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:49 smithi150 systemd[1]: Stopped Ceph mon.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:15:49.116 INFO:journalctl@ceph.mon.a.smithi150.stdout:Apr 05 03:15:49 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.a.service: Consumed 20.117s CPU time. 2024-04-05T03:15:49.148 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:15:49.148 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-05T03:15:49.148 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-05T03:15:49.148 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.c 2024-04-05T03:15:49.476 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:49 smithi150 systemd[1]: Stopping Ceph mon.c for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:15:49.962 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:03:15:49] ENGINE Bus STOPPING 2024-04-05T03:15:49.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:03:15:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-05T03:15:49.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:03:15:49] ENGINE Bus STOPPED 2024-04-05T03:15:49.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:03:15:49] ENGINE Bus STARTING 2024-04-05T03:15:49.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:03:15:49] ENGINE Serving on http://:::9283 2024-04-05T03:15:49.963 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: [05/Apr/2024:03:15:49] ENGINE Bus STARTED 2024-04-05T03:15:49.963 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c[107664]: 2024-04-05T03:15:49.506+0000 7f7fe4291700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:15:49.964 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:49 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c[107664]: 2024-04-05T03:15:49.506+0000 7f7fe4291700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-05T03:15:49.964 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:49 smithi150 podman[165508]: 2024-04-05 03:15:49.702372552 +0000 UTC m=+0.332607748 container died 74316b4d98ae1d1cfbefb4d99a35e704a5027422cb578c2458d77437c175bc76 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-05T03:15:50.226 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:50 smithi150 podman[165508]: 2024-04-05 03:15:50.155004559 +0000 UTC m=+0.785239755 container cleanup 74316b4d98ae1d1cfbefb4d99a35e704a5027422cb578c2458d77437c175bc76 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c, org.label-schema.build-date=20240403, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-05T03:15:50.227 INFO:journalctl@ceph.mon.c.smithi150.stdout:Apr 05 03:15:50 smithi150 bash[165508]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-c 2024-04-05T03:15:50.565 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.c.service' 2024-04-05T03:15:50.601 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:15:50.601 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-05T03:15:50.601 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-05T03:15:50.602 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.b 2024-04-05T03:15:50.885 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:50 smithi196 systemd[1]: Stopping Ceph mon.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:15:51.221 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b[91328]: 2024-04-05T03:15:50.932+0000 7f2ab5b8b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:15:51.221 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b[91328]: 2024-04-05T03:15:50.932+0000 7f2ab5b8b700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-05T03:15:51.221 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:50 smithi196 podman[105228]: 2024-04-05 03:15:50.974166709 +0000 UTC m=+0.171998362 container died 51bbb41f1c91054951f544fa05101ef046fba67950e1489e1d413442cf14a6b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True) 2024-04-05T03:15:51.621 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:51 smithi196 podman[105228]: 2024-04-05 03:15:51.41386848 +0000 UTC m=+0.611700133 container cleanup 51bbb41f1c91054951f544fa05101ef046fba67950e1489e1d413442cf14a6b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:15:51.622 INFO:journalctl@ceph.mon.b.smithi196.stdout:Apr 05 03:15:51 smithi196 bash[105228]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mon-b 2024-04-05T03:15:51.823 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mon.b.service' 2024-04-05T03:15:51.868 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:15:51.868 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-05T03:15:51.868 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-05T03:15:51.869 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y 2024-04-05T03:15:51.897 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:51 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: ::ffff:172.21.15.196 - - [05/Apr/2024:03:15:51] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.43.0" 2024-04-05T03:15:52.185 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:51 smithi150 systemd[1]: Stopping Ceph mgr.y for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:15:52.476 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:15:52.185+0000 7f46062c2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:15:52.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:52 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y[92715]: 2024-04-05T03:15:52.185+0000 7f46062c2700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-05T03:15:52.477 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:52 smithi150 podman[165652]: 2024-04-05 03:15:52.218385013 +0000 UTC m=+0.155428247 container died 0d5b82cc2eb247424834a171d45a00eb9d8eb40ef3c40b80809577b1788e658e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, ceph=True) 2024-04-05T03:15:52.867 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:52 smithi150 podman[165652]: 2024-04-05 03:15:52.618244022 +0000 UTC m=+0.555287290 container cleanup 0d5b82cc2eb247424834a171d45a00eb9d8eb40ef3c40b80809577b1788e658e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:15:52.868 INFO:journalctl@ceph.mgr.y.smithi150.stdout:Apr 05 03:15:52 smithi150 bash[165652]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-y 2024-04-05T03:15:53.008 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.y.service' 2024-04-05T03:15:53.085 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:15:53.086 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-05T03:15:53.086 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-05T03:15:53.086 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x 2024-04-05T03:15:53.385 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 03:15:53 smithi196 systemd[1]: Stopping Ceph mgr.x for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:15:53.734 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 03:15:53 smithi196 podman[105360]: 2024-04-05 03:15:53.435418294 +0000 UTC m=+0.156234843 container died 7ca2051bb3927143852d92b48b807b457cd887a349e234ea4a4ffdb5df52af66 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-05T03:15:54.017 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 03:15:53 smithi196 podman[105360]: 2024-04-05 03:15:53.809967316 +0000 UTC m=+0.530783910 container cleanup 7ca2051bb3927143852d92b48b807b457cd887a349e234ea4a4ffdb5df52af66 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-04-05T03:15:54.017 INFO:journalctl@ceph.mgr.x.smithi196.stdout:Apr 05 03:15:53 smithi196 bash[105360]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-mgr-x 2024-04-05T03:15:54.219 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@mgr.x.service' 2024-04-05T03:15:54.257 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:15:54.257 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-05T03:15:54.257 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-05T03:15:54.257 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.0 2024-04-05T03:15:54.541 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:15:54 smithi150 systemd[1]: Stopping Ceph osd.0 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:15:54.976 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:15:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[117509]: 2024-04-05T03:15:54.542+0000 7f7016bb5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:15:54.976 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:15:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[117509]: 2024-04-05T03:15:54.542+0000 7f7016bb5700 -1 osd.0 140 *** Got signal Terminated *** 2024-04-05T03:15:54.977 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:15:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0[117509]: 2024-04-05T03:15:54.542+0000 7f7016bb5700 -1 osd.0 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:00.227 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:15:59 smithi150 podman[165785]: 2024-04-05 03:15:59.963564886 +0000 UTC m=+5.519592788 container died 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, RELEASE=squid-0028525, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:00.689 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:00 smithi150 podman[165785]: 2024-04-05 03:16:00.414308609 +0000 UTC m=+5.970336552 container cleanup 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, GIT_BRANCH=HEAD, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:00.689 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:00 smithi150 bash[165785]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0 2024-04-05T03:16:00.976 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:00 smithi150 podman[165809]: 2024-04-05 03:16:00.690639713 +0000 UTC m=+0.721435093 container remove 415d151c052dffddb1866caed3d3c2b7466728fa982d254051ee11ac3d95011e (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:01.342 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:01 smithi150 podman[165905]: 2024-04-05 03:16:00.927222608 +0000 UTC m=+0.017311110 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:16:01.342 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:01 smithi150 podman[165905]: 2024-04-05 03:16:01.033468582 +0000 UTC m=+0.123557066 container create fcbfb24348ffc429060c0f43c413b8567e88dab40283a8e899da87e2efa725b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-05T03:16:01.342 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:01 smithi150 podman[165905]: 2024-04-05 03:16:01.235754227 +0000 UTC m=+0.325842712 container init fcbfb24348ffc429060c0f43c413b8567e88dab40283a8e899da87e2efa725b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-0028525, io.buildah.version=1.33.5, org.label-schema.build-date=20240403) 2024-04-05T03:16:01.343 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:01 smithi150 podman[165905]: 2024-04-05 03:16:01.241247704 +0000 UTC m=+0.331336194 container start fcbfb24348ffc429060c0f43c413b8567e88dab40283a8e899da87e2efa725b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-0028525, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:16:01.343 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:01 smithi150 podman[165905]: 2024-04-05 03:16:01.343997722 +0000 UTC m=+0.434086212 container attach fcbfb24348ffc429060c0f43c413b8567e88dab40283a8e899da87e2efa725b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-04-05T03:16:01.726 INFO:journalctl@ceph.osd.0.smithi150.stdout:Apr 05 03:16:01 smithi150 podman[165905]: 2024-04-05 03:16:01.53267481 +0000 UTC m=+0.622763291 container died fcbfb24348ffc429060c0f43c413b8567e88dab40283a8e899da87e2efa725b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-0-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-0028525, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:02.248 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.0.service' 2024-04-05T03:16:02.291 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:02.292 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-05T03:16:02.292 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-05T03:16:02.292 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1 2024-04-05T03:16:02.636 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:02 smithi150 systemd[1]: Stopping Ceph osd.1 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:02.976 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[126597]: 2024-04-05T03:16:02.636+0000 7f071e90a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:02.977 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[126597]: 2024-04-05T03:16:02.636+0000 7f071e90a700 -1 osd.1 140 *** Got signal Terminated *** 2024-04-05T03:16:02.977 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:02 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1[126597]: 2024-04-05T03:16:02.636+0000 7f071e90a700 -1 osd.1 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:04.726 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:04.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:04.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:04 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:04.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:08.435 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:08 smithi150 podman[166035]: 2024-04-05 03:16:08.129068892 +0000 UTC m=+5.615488525 container died 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-0028525, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , ceph=True) 2024-04-05T03:16:08.726 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:08 smithi150 podman[166035]: 2024-04-05 03:16:08.634177751 +0000 UTC m=+6.120597391 container cleanup 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, ceph=True, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:16:08.726 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:08 smithi150 bash[166035]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1 2024-04-05T03:16:09.149 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:08 smithi150 podman[166060]: 2024-04-05 03:16:08.844001147 +0000 UTC m=+0.713669639 container remove 033996a9bd5940d854bb5b6ed92dee93a8e6c1f5ce768c11eaa3864d4827fa54 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0) 2024-04-05T03:16:09.461 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:09 smithi150 podman[166159]: 2024-04-05 03:16:09.054149923 +0000 UTC m=+0.018126218 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:16:09.462 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:09 smithi150 podman[166159]: 2024-04-05 03:16:09.1613182 +0000 UTC m=+0.125294489 container create e399d677968b5f6e7d9bd34106035404eb0c1f4628d44f56ba26a1470a3a85c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-04-05T03:16:09.462 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:09 smithi150 podman[166159]: 2024-04-05 03:16:09.355714985 +0000 UTC m=+0.319691282 container init e399d677968b5f6e7d9bd34106035404eb0c1f4628d44f56ba26a1470a3a85c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=squid-0028525, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.build-date=20240403) 2024-04-05T03:16:09.462 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:09 smithi150 podman[166159]: 2024-04-05 03:16:09.36208622 +0000 UTC m=+0.326062510 container start e399d677968b5f6e7d9bd34106035404eb0c1f4628d44f56ba26a1470a3a85c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-0028525, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:16:09.462 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:09 smithi150 podman[166159]: 2024-04-05 03:16:09.463500173 +0000 UTC m=+0.427476478 container attach e399d677968b5f6e7d9bd34106035404eb0c1f4628d44f56ba26a1470a3a85c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403) 2024-04-05T03:16:09.726 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:09 smithi150 podman[166159]: 2024-04-05 03:16:09.658852056 +0000 UTC m=+0.622828341 container died e399d677968b5f6e7d9bd34106035404eb0c1f4628d44f56ba26a1470a3a85c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-0028525, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:10.189 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1.service' 2024-04-05T03:16:10.216 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:10 smithi150 podman[166204]: 2024-04-05 03:16:10.175286438 +0000 UTC m=+0.510942130 container remove e399d677968b5f6e7d9bd34106035404eb0c1f4628d44f56ba26a1470a3a85c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, RELEASE=squid-0028525, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:10.216 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:10 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1.service: Deactivated successfully. 2024-04-05T03:16:10.216 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:10 smithi150 systemd[1]: Stopped Ceph osd.1 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:16:10.216 INFO:journalctl@ceph.osd.1.smithi150.stdout:Apr 05 03:16:10 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.1.service: Consumed 8min 23.083s CPU time. 2024-04-05T03:16:10.217 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:10.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:10.217 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:10.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:10.232 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:10.232 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-05T03:16:10.232 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-05T03:16:10.232 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.2 2024-04-05T03:16:10.476 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:10 smithi150 systemd[1]: Stopping Ceph osd.2 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:10.976 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[136515]: 2024-04-05T03:16:10.588+0000 7f94afd20700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:10.977 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[136515]: 2024-04-05T03:16:10.588+0000 7f94afd20700 -1 osd.2 140 *** Got signal Terminated *** 2024-04-05T03:16:10.977 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:10 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2[136515]: 2024-04-05T03:16:10.588+0000 7f94afd20700 -1 osd.2 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:13.636 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:13.161Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-05T03:16:13.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:13.161Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-05T03:16:13.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:13.162Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-05T03:16:13.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:13.162Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-05T03:16:13.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:13 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:13.164Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.150:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.150:8765: connect: connection refused" 2024-04-05T03:16:14.581 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:14 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:14.311Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:14.581 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:14 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:14.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:14.581 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:14 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:14.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:16.226 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:16 smithi150 podman[166288]: 2024-04-05 03:16:16.042654982 +0000 UTC m=+5.585605343 container died 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-05T03:16:16.652 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:16 smithi150 podman[166288]: 2024-04-05 03:16:16.461249595 +0000 UTC m=+6.004199957 container cleanup 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-0028525, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-05T03:16:16.653 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:16 smithi150 bash[166288]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2 2024-04-05T03:16:16.976 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:16 smithi150 podman[166322]: 2024-04-05 03:16:16.65422074 +0000 UTC m=+0.604803781 container remove 155b19d15ab1b9db350d1019780b02cbabb895689c1ea0ef9973d5ee36dfdccb (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux ) 2024-04-05T03:16:17.273 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:16 smithi150 podman[166419]: 2024-04-05 03:16:16.898094875 +0000 UTC m=+0.017766381 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:16:17.273 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:17 smithi150 podman[166419]: 2024-04-05 03:16:17.013593191 +0000 UTC m=+0.133264761 container create dcdbe566ad4675dab6c4dbc5c47251e1f9062d37230878f8a028b4f83384545c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, ceph=True, CEPH_POINT_RELEASE=) 2024-04-05T03:16:17.273 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:17 smithi150 podman[166419]: 2024-04-05 03:16:17.274768731 +0000 UTC m=+0.394440236 container init dcdbe566ad4675dab6c4dbc5c47251e1f9062d37230878f8a028b4f83384545c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-0028525, org.label-schema.license=GPLv2) 2024-04-05T03:16:17.575 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:17 smithi150 podman[166419]: 2024-04-05 03:16:17.280281399 +0000 UTC m=+0.399952904 container start dcdbe566ad4675dab6c4dbc5c47251e1f9062d37230878f8a028b4f83384545c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, RELEASE=squid-0028525, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-05T03:16:17.576 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:17 smithi150 podman[166419]: 2024-04-05 03:16:17.39117388 +0000 UTC m=+0.510845398 container attach dcdbe566ad4675dab6c4dbc5c47251e1f9062d37230878f8a028b4f83384545c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240403) 2024-04-05T03:16:17.876 INFO:journalctl@ceph.osd.2.smithi150.stdout:Apr 05 03:16:17 smithi150 podman[166419]: 2024-04-05 03:16:17.577539872 +0000 UTC m=+0.697211379 container died dcdbe566ad4675dab6c4dbc5c47251e1f9062d37230878f8a028b4f83384545c (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403) 2024-04-05T03:16:18.238 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.2.service' 2024-04-05T03:16:18.281 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:18.281 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-05T03:16:18.281 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-05T03:16:18.282 DEBUG:teuthology.orchestra.run.smithi150:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.3 2024-04-05T03:16:18.476 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:18 smithi150 systemd[1]: Stopping Ceph osd.3 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:18.976 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:18 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:16:18.615+0000 7f77cc9d0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:18.977 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:18 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:16:18.615+0000 7f77cc9d0700 -1 osd.3 140 *** Got signal Terminated *** 2024-04-05T03:16:18.977 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:18 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:16:18.615+0000 7f77cc9d0700 -1 osd.3 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:20.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:20.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:20.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:20.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:20.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:20 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:20.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:20.636 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:20.161+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:21.151 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:20 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:20.895+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:21.635 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:21 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:21.153+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:22.386 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:22.138+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:22.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:21 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:21.929+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:23.126 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:22.906+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:23.126 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:22 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:22.659+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:23.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:23.127+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:23.726 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:23 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3[141774]: 2024-04-05T03:16:23.386+0000 7f77c8fc4700 -1 osd.3 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:59.337673+0000 front 2024-04-05T03:15:59.337619+0000 (oldest deadline 2024-04-05T03:16:22.837718+0000) 2024-04-05T03:16:24.073 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:23.682+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:24.073 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:23 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:23.912+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:24.339 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:24.155+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:24.339 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:24.073+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:24.430 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:24 smithi150 podman[166549]: 2024-04-05 03:16:24.164506863 +0000 UTC m=+5.647453907 container died b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-0028525, io.buildah.version=1.33.5) 2024-04-05T03:16:24.430 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:24.312Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:24.431 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:24.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:24.431 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:24 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:24.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:24.726 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:24 smithi150 podman[166549]: 2024-04-05 03:16:24.638768583 +0000 UTC m=+6.121715626 container cleanup b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, org.label-schema.schema-version=1.0) 2024-04-05T03:16:24.726 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:24 smithi150 bash[166549]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3 2024-04-05T03:16:25.036 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:24.898+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:25.036 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:24 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:24.732+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:25.226 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:24 smithi150 podman[166571]: 2024-04-05 03:16:24.898558064 +0000 UTC m=+0.733471402 container remove b1a0d7c6725e4e0c11791c50c880727b8a3bf8803251505daf21e166d2d04722 (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-0028525, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-04-05T03:16:25.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:25.134+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:25.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:25.037+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:25.542 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:25 smithi150 podman[166666]: 2024-04-05 03:16:25.249264308 +0000 UTC m=+0.114418140 container create dc8ac5e180149c6b339ebb63b0ebc25ac8a98f33d86ff3d4801aaf16927c71be (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-05T03:16:25.542 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:25 smithi150 podman[166666]: 2024-04-05 03:16:25.15292993 +0000 UTC m=+0.018083733 image pull d8c35eaf3c25b624514c7b18d9f1d2cedc1bbdfe6c87a522d661faa4038ccad8 quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca 2024-04-05T03:16:25.542 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:25 smithi150 podman[166666]: 2024-04-05 03:16:25.459882047 +0000 UTC m=+0.325035837 container init dc8ac5e180149c6b339ebb63b0ebc25ac8a98f33d86ff3d4801aaf16927c71be (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:16:25.542 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:25 smithi150 podman[166666]: 2024-04-05 03:16:25.4652654 +0000 UTC m=+0.330419193 container start dc8ac5e180149c6b339ebb63b0ebc25ac8a98f33d86ff3d4801aaf16927c71be (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, RELEASE=squid-0028525, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403, GIT_CLEAN=True) 2024-04-05T03:16:25.542 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:25 smithi150 podman[166666]: 2024-04-05 03:16:25.542976633 +0000 UTC m=+0.408130427 container attach dc8ac5e180149c6b339ebb63b0ebc25ac8a98f33d86ff3d4801aaf16927c71be (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-0028525, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, io.buildah.version=1.33.5) 2024-04-05T03:16:25.976 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:25 smithi150 podman[166666]: 2024-04-05 03:16:25.757178741 +0000 UTC m=+0.622332538 container died dc8ac5e180149c6b339ebb63b0ebc25ac8a98f33d86ff3d4801aaf16927c71be (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=squid-0028525) 2024-04-05T03:16:26.068 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:25.761+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:26.068 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:25 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:25.861+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:26.311 INFO:journalctl@ceph.osd.3.smithi150.stdout:Apr 05 03:16:26 smithi150 podman[166712]: 2024-04-05 03:16:26.29580275 +0000 UTC m=+0.533252260 container remove dc8ac5e180149c6b339ebb63b0ebc25ac8a98f33d86ff3d4801aaf16927c71be (image=quay.ceph.io/ceph-ci/ceph@sha256:b1ac2def9baea35d725102f90134220b12af7a73c65adaf7747c4c8e1408c7ca, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-0028525, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-05T03:16:26.316 DEBUG:teuthology.orchestra.run.smithi150:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.3.service' 2024-04-05T03:16:26.355 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:26.356 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-05T03:16:26.356 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-05T03:16:26.356 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.4 2024-04-05T03:16:26.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:26.112+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:26.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:26.069+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:26.654 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:26 smithi196 systemd[1]: Stopping Ceph osd.4 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:27.071 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:26.783+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:27.071 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:26.878+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:27.072 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:26.654+0000 7f7a9c141700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:27.072 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:26.654+0000 7f7a9c141700 -1 osd.4 140 *** Got signal Terminated *** 2024-04-05T03:16:27.072 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:26 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:26.654+0000 7f7a9c141700 -1 osd.4 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:27.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:27.116+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:27.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:27.072+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:28.065 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:27.905+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:28.066 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:27 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:27.757+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:28.066 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:28.067+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:28.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:28.135+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:29.052 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:28.941+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:29.052 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:28.755+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:29.053 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:28 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:28.755+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:29.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:29.053+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:29.386 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:29.116+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:30.083 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:29.805+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:30.083 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:29.805+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:30.084 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:29.894+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:30.084 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:29 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:29.894+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:30.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:30.122+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:30.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:30.084+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:30.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:30.084+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:30.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:30.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:30.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:30.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:30.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:30 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:30.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:31.094 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:30.844+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:31.094 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:30.844+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:31.094 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:30.925+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:31.095 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:30 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:30.925+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:31.385 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4[38702]: 2024-04-05T03:16:31.095+0000 7f7a9673b700 -1 osd.4 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:58.321846+0000 front 2024-04-05T03:15:58.321657+0000 (oldest deadline 2024-04-05T03:16:20.021577+0000) 2024-04-05T03:16:31.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:31.097+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:31.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:31.097+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:32.135 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:32 smithi196 podman[105505]: 2024-04-05 03:16:32.089522211 +0000 UTC m=+5.535828043 container died b2180320941570ef03dbb94f2b28de35aa45401b1c89ecd4726cb5ea5c59d8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403) 2024-04-05T03:16:32.136 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:31.884+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:32.136 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:31.884+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:32.136 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:32.096+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:32.136 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:32.096+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:32.137 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:31.910+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:32.137 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:31 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:31.910+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:32.635 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:32 smithi196 podman[105505]: 2024-04-05 03:16:32.468293996 +0000 UTC m=+5.914599817 container cleanup b2180320941570ef03dbb94f2b28de35aa45401b1c89ecd4726cb5ea5c59d8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:32.636 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:32 smithi196 bash[105505]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4 2024-04-05T03:16:33.068 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:32 smithi196 podman[105528]: 2024-04-05 03:16:32.719921446 +0000 UTC m=+0.632310495 container remove b2180320941570ef03dbb94f2b28de35aa45401b1c89ecd4726cb5ea5c59d8ac (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2) 2024-04-05T03:16:33.068 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:32.901+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:33.068 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:32.901+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:33.068 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:32.915+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:33.069 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:32 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:32.915+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:33.380 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:33.068+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:33.380 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:33.068+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:33.380 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:33 smithi196 podman[105625]: 2024-04-05 03:16:32.980277577 +0000 UTC m=+0.017863544 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:16:33.380 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:33 smithi196 podman[105625]: 2024-04-05 03:16:33.078973784 +0000 UTC m=+0.116559743 container create 07ad2f4024952da9253c31533f898ecc48c5139553e95a3749d777f70cb01181 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-05T03:16:33.380 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:33 smithi196 podman[105625]: 2024-04-05 03:16:33.289774167 +0000 UTC m=+0.327360119 container init 07ad2f4024952da9253c31533f898ecc48c5139553e95a3749d777f70cb01181 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0) 2024-04-05T03:16:33.381 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:33 smithi196 podman[105625]: 2024-04-05 03:16:33.295627449 +0000 UTC m=+0.333213412 container start 07ad2f4024952da9253c31533f898ecc48c5139553e95a3749d777f70cb01181 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4-deactivate, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True) 2024-04-05T03:16:33.381 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:33 smithi196 podman[105625]: 2024-04-05 03:16:33.381563384 +0000 UTC m=+0.419149349 container attach 07ad2f4024952da9253c31533f898ecc48c5139553e95a3749d777f70cb01181 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4-deactivate, GIT_CLEAN=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-05T03:16:33.635 INFO:journalctl@ceph.osd.4.smithi196.stdout:Apr 05 03:16:33 smithi196 podman[105625]: 2024-04-05 03:16:33.592198606 +0000 UTC m=+0.629784563 container died 07ad2f4024952da9253c31533f898ecc48c5139553e95a3749d777f70cb01181 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-05T03:16:34.133 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:33.932+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:34.133 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:33.932+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:34.133 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:33.898+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:34.134 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:33 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:33.898+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:34.134 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:34.083+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:34.134 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:34.083+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:34.162 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.4.service' 2024-04-05T03:16:34.243 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:34.244 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-05T03:16:34.244 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-05T03:16:34.244 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.5 2024-04-05T03:16:34.385 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:34 smithi196 systemd[1]: Stopping Ceph osd.5 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:34.726 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:34 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:34.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:34.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:34 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:34.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:34.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:34 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:34.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:34.885 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:34.871+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:34.886 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:34.871+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:34.886 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:34.631+0000 7fdea0716700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:34.886 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:34.632+0000 7fdea0716700 -1 osd.5 140 *** Got signal Terminated *** 2024-04-05T03:16:34.886 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:34.632+0000 7fdea0716700 -1 osd.5 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:35.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:35.096+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:35.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:35.096+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:35.386 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:34.890+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:35.386 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:34 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:34.890+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:36.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:35.885+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:36.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:35.885+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:36.137 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:35.875+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:36.137 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:35 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:35.875+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:36.137 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:36 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:36.068+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:36.137 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:36 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:36.068+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:37.106 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:36 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:36.903+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:37.106 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:36 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:36.904+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:37.106 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:36 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:36.853+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:37.106 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:36 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:36.853+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:37.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:37.106+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:37.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:37.106+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:38.114 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:37.858+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:38.114 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:37.858+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:38.114 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:37.858+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:38.114 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:37.900+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:38.115 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:37.900+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:38.115 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:37 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:37.900+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:15.264627+0000 front 2024-04-05T03:16:15.264588+0000 (oldest deadline 2024-04-05T03:16:37.564533+0000) 2024-04-05T03:16:38.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:38.114+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:38.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:38.114+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:39.135 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:38.863+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.261600+0000 front 2024-04-05T03:15:57.261729+0000 (oldest deadline 2024-04-05T03:16:22.561696+0000) 2024-04-05T03:16:39.136 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:38.863+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:03.062684+0000 front 2024-04-05T03:16:03.062763+0000 (oldest deadline 2024-04-05T03:16:28.362647+0000) 2024-04-05T03:16:39.136 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5[44629]: 2024-04-05T03:16:38.863+0000 7fde9dd16700 -1 osd.5 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:15.264627+0000 front 2024-04-05T03:16:15.264588+0000 (oldest deadline 2024-04-05T03:16:37.564533+0000) 2024-04-05T03:16:39.136 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:39 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:39.081+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:39.136 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:39 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:39.081+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:39.136 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:39 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:39.081+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:39.137 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:38.867+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:39.137 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:38.867+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:39.137 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:38 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:38.867+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:40.092 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:40 smithi196 podman[105755]: 2024-04-05 03:16:40.060460033 +0000 UTC m=+5.588198441 container died 5415e890a8702db9c12aa57db36da69b596a79977b4b5e64cdb4bbba70ecdc21 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:40.093 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:39 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:39.836+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:40.093 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:39 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:39.836+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:40.093 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:39 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:39.836+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:40.345 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:40.093+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:40.346 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:40.093+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:40.346 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:40.093+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:40.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:40.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:40.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:40.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:40.477 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:40 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:40.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:40.635 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:40 smithi196 podman[105755]: 2024-04-05 03:16:40.4460463 +0000 UTC m=+5.973784724 container cleanup 5415e890a8702db9c12aa57db36da69b596a79977b4b5e64cdb4bbba70ecdc21 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.build-date=20240403) 2024-04-05T03:16:40.636 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:40 smithi196 bash[105755]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5 2024-04-05T03:16:41.056 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:40 smithi196 podman[105779]: 2024-04-05 03:16:40.72277053 +0000 UTC m=+0.660519956 container remove 5415e890a8702db9c12aa57db36da69b596a79977b4b5e64cdb4bbba70ecdc21 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:41.056 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:40.845+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:41.056 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:40.845+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:41.057 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:40 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:40.845+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:41.374 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:41 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:41.057+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:41.374 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:41 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:41.057+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:41.374 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:41 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:41.057+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:41.375 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:41 smithi196 podman[105878]: 2024-04-05 03:16:40.976489946 +0000 UTC m=+0.018359821 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:16:41.375 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:41 smithi196 podman[105878]: 2024-04-05 03:16:41.081873167 +0000 UTC m=+0.123743040 container create 7a95208b6b79457f398f4796218d43c1267fc60ed212cd0b38c2d8f05a8e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:41.375 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:41 smithi196 podman[105878]: 2024-04-05 03:16:41.29265077 +0000 UTC m=+0.334520644 container init 7a95208b6b79457f398f4796218d43c1267fc60ed212cd0b38c2d8f05a8e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240403) 2024-04-05T03:16:41.375 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:41 smithi196 podman[105878]: 2024-04-05 03:16:41.298650316 +0000 UTC m=+0.340520190 container start 7a95208b6b79457f398f4796218d43c1267fc60ed212cd0b38c2d8f05a8e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-04-05T03:16:41.375 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:41 smithi196 podman[105878]: 2024-04-05 03:16:41.3758228 +0000 UTC m=+0.417692681 container attach 7a95208b6b79457f398f4796218d43c1267fc60ed212cd0b38c2d8f05a8e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5-deactivate, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2) 2024-04-05T03:16:41.635 INFO:journalctl@ceph.osd.5.smithi196.stdout:Apr 05 03:16:41 smithi196 podman[105878]: 2024-04-05 03:16:41.581084847 +0000 UTC m=+0.622954724 container died 7a95208b6b79457f398f4796218d43c1267fc60ed212cd0b38c2d8f05a8e94c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-04-05T03:16:42.127 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:42.083+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:42.127 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:42.083+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:42.127 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:42.083+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:42.128 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:41 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:41.871+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:42.128 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:41 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:41.871+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:42.128 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:41 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:41.871+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:42.153 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.5.service' 2024-04-05T03:16:42.193 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:42.194 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-05T03:16:42.194 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-05T03:16:42.194 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.6 2024-04-05T03:16:42.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 systemd[1]: Stopping Ceph osd.6 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:42.885 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:42.517+0000 7f4fed55e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:42.886 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:42.517+0000 7f4fed55e700 -1 osd.6 140 *** Got signal Terminated *** 2024-04-05T03:16:42.886 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:42.517+0000 7f4fed55e700 -1 osd.6 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:43.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:43.087+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:43.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:43.087+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:43.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:43.087+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:43.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:42.916+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:43.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:42.916+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:43.387 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:42 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:42.916+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:44.086 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:43.883+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:44.087 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:43.883+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:44.087 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:43 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:43.883+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:44.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:44.087+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:44.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:44.087+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:44.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:44.087+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:44.726 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:44.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:44.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:44.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:44.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:44 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:44.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:45.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:45.048+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:45.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:45.048+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:45.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:45.048+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:45.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:44.900+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:45.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:44.900+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:45.387 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:44 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:44.900+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:46.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:46.012+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:46.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:46.012+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:46.387 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:46.012+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:46.387 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:45.945+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:46.388 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:45.945+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:46.388 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:45 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:45.945+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:47.385 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:46.997+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:57.004760+0000 front 2024-04-05T03:15:57.004873+0000 (oldest deadline 2024-04-05T03:16:24.005375+0000) 2024-04-05T03:16:47.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:46.997+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.006126+0000 front 2024-04-05T03:16:04.005996+0000 (oldest deadline 2024-04-05T03:16:29.305877+0000) 2024-04-05T03:16:47.386 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6[50855]: 2024-04-05T03:16:46.997+0000 7f4feb35f700 -1 osd.6 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:13.307353+0000 front 2024-04-05T03:16:13.307372+0000 (oldest deadline 2024-04-05T03:16:38.607260+0000) 2024-04-05T03:16:47.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:46.971+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:47.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:46.971+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:47.386 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:46 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:46.971+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:48.214 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:47 smithi196 podman[106009]: 2024-04-05 03:16:47.946168404 +0000 UTC m=+5.529622047 container died 85a4b25b62cf628bc4fbb69e263a158d9e2ab5010757d1687aa58091b5cb774b (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=quincy-6fda41f, ceph=True, org.label-schema.build-date=20240403, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-04-05T03:16:48.214 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:47.948+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:48.215 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:47.948+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:48.215 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:47 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:47.948+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:48.565 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:48 smithi196 podman[106009]: 2024-04-05 03:16:48.323369366 +0000 UTC m=+5.906823009 container cleanup 85a4b25b62cf628bc4fbb69e263a158d9e2ab5010757d1687aa58091b5cb774b (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-04-05T03:16:48.565 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:48 smithi196 bash[106009]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6 2024-04-05T03:16:48.885 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:48 smithi196 podman[106043]: 2024-04-05 03:16:48.566313453 +0000 UTC m=+0.617540366 container remove 85a4b25b62cf628bc4fbb69e263a158d9e2ab5010757d1687aa58091b5cb774b (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6, org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:49.217 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:48.903+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:49.217 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:48.903+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:49.218 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:48 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:48.903+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:49.218 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:48 smithi196 podman[106144]: 2024-04-05 03:16:48.798503223 +0000 UTC m=+0.018202386 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:16:49.218 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:48 smithi196 podman[106144]: 2024-04-05 03:16:48.90053611 +0000 UTC m=+0.120235314 container create 4363f1680f0cc4933b3c9f6d50d28c00cf59f32a5e0f0150d3e7120d6e7c89bd (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:49.218 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:49 smithi196 podman[106144]: 2024-04-05 03:16:49.127979805 +0000 UTC m=+0.347678967 container init 4363f1680f0cc4933b3c9f6d50d28c00cf59f32a5e0f0150d3e7120d6e7c89bd (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6-deactivate, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240403, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-6fda41f) 2024-04-05T03:16:49.218 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:49 smithi196 podman[106144]: 2024-04-05 03:16:49.133481739 +0000 UTC m=+0.353180903 container start 4363f1680f0cc4933b3c9f6d50d28c00cf59f32a5e0f0150d3e7120d6e7c89bd (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-05T03:16:49.218 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:49 smithi196 podman[106144]: 2024-04-05 03:16:49.219450405 +0000 UTC m=+0.439149574 container attach 4363f1680f0cc4933b3c9f6d50d28c00cf59f32a5e0f0150d3e7120d6e7c89bd (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-6fda41f, org.label-schema.build-date=20240403, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-05T03:16:49.635 INFO:journalctl@ceph.osd.6.smithi196.stdout:Apr 05 03:16:49 smithi196 podman[106144]: 2024-04-05 03:16:49.409764544 +0000 UTC m=+0.629463698 container died 4363f1680f0cc4933b3c9f6d50d28c00cf59f32a5e0f0150d3e7120d6e7c89bd (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f) 2024-04-05T03:16:50.029 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:49.854+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:50.029 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:49.854+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:50.030 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:49.854+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:50.030 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:49 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:49.854+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6830 osd.3 since back 2024-04-05T03:16:18.415400+0000 front 2024-04-05T03:16:18.415413+0000 (oldest deadline 2024-04-05T03:16:49.015647+0000) 2024-04-05T03:16:50.050 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.6.service' 2024-04-05T03:16:50.091 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:50.092 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-05T03:16:50.092 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-05T03:16:50.092 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.7 2024-04-05T03:16:50.385 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 systemd[1]: Stopping Ceph osd.7 for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:50.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:50.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:50.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:50.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:50.476 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:50 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:50.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:50.821 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.426+0000 7f0cfb3aa700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-05T03:16:50.821 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.426+0000 7f0cfb3aa700 -1 osd.7 140 *** Got signal Terminated *** 2024-04-05T03:16:50.821 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.426+0000 7f0cfb3aa700 -1 osd.7 140 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-05T03:16:50.821 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.822+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:50.821 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.822+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:50.822 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.822+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:50.822 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:50 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:50.822+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6830 osd.3 since back 2024-04-05T03:16:18.415400+0000 front 2024-04-05T03:16:18.415413+0000 (oldest deadline 2024-04-05T03:16:49.015647+0000) 2024-04-05T03:16:52.135 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:51.781+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:52.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:51.781+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:52.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:51.781+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:52.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:51 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:51.781+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6830 osd.3 since back 2024-04-05T03:16:18.415400+0000 front 2024-04-05T03:16:18.415413+0000 (oldest deadline 2024-04-05T03:16:49.015647+0000) 2024-04-05T03:16:53.135 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:52.795+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:53.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:52.795+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:53.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:52.795+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:53.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:52 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:52.795+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6830 osd.3 since back 2024-04-05T03:16:18.415400+0000 front 2024-04-05T03:16:18.415413+0000 (oldest deadline 2024-04-05T03:16:49.015647+0000) 2024-04-05T03:16:54.090 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:53.778+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:54.090 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:53.778+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:54.090 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:53.778+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:54.090 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:53.778+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6830 osd.3 since back 2024-04-05T03:16:18.415400+0000 front 2024-04-05T03:16:18.415413+0000 (oldest deadline 2024-04-05T03:16:49.015647+0000) 2024-04-05T03:16:54.090 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:53 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:53.778+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.196:6804 osd.4 since back 2024-04-05T03:16:29.016294+0000 front 2024-04-05T03:16:29.016306+0000 (oldest deadline 2024-04-05T03:16:53.116185+0000) 2024-04-05T03:16:54.726 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:54.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:54.726 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:54.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:16:54.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:16:54 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:16:54.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:16:55.135 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:54.758+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6806 osd.0 since back 2024-04-05T03:15:56.312080+0000 front 2024-04-05T03:15:56.312144+0000 (oldest deadline 2024-04-05T03:16:20.412034+0000) 2024-04-05T03:16:55.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:54.758+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6814 osd.1 since back 2024-04-05T03:16:04.513187+0000 front 2024-04-05T03:16:04.513204+0000 (oldest deadline 2024-04-05T03:16:29.813071+0000) 2024-04-05T03:16:55.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:54.758+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6822 osd.2 since back 2024-04-05T03:16:14.414268+0000 front 2024-04-05T03:16:14.414331+0000 (oldest deadline 2024-04-05T03:16:37.314209+0000) 2024-04-05T03:16:55.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:54.758+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.150:6830 osd.3 since back 2024-04-05T03:16:18.415400+0000 front 2024-04-05T03:16:18.415413+0000 (oldest deadline 2024-04-05T03:16:49.015647+0000) 2024-04-05T03:16:55.136 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:54 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7[56973]: 2024-04-05T03:16:54.758+0000 7f0cf59a4700 -1 osd.7 140 heartbeat_check: no reply from 172.21.15.196:6804 osd.4 since back 2024-04-05T03:16:29.016294+0000 front 2024-04-05T03:16:29.016306+0000 (oldest deadline 2024-04-05T03:16:53.116185+0000) 2024-04-05T03:16:56.135 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:55 smithi196 podman[106275]: 2024-04-05 03:16:55.879714323 +0000 UTC m=+5.560950748 container died ed60a8dad4f7dd343829f007d4daa535d2623c4b714129020c8903e9da2beaee (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:56.491 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:56 smithi196 podman[106275]: 2024-04-05 03:16:56.241538415 +0000 UTC m=+5.922774834 container cleanup ed60a8dad4f7dd343829f007d4daa535d2623c4b714129020c8903e9da2beaee (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240403, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-6fda41f, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:16:56.491 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:56 smithi196 bash[106275]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7 2024-04-05T03:16:56.841 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:56 smithi196 podman[106299]: 2024-04-05 03:16:56.49285625 +0000 UTC m=+0.614852622 container remove ed60a8dad4f7dd343829f007d4daa535d2623c4b714129020c8903e9da2beaee (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7, ceph=True, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-6fda41f, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-05T03:16:57.092 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:56 smithi196 podman[106399]: 2024-04-05 03:16:56.843495398 +0000 UTC m=+0.110628273 container create a28bbc73aeed49a2fb6c991d9bad3dd7d3e388af8e3be98bc74c55428d80cfe3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-6fda41f, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0) 2024-04-05T03:16:57.092 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:56 smithi196 podman[106399]: 2024-04-05 03:16:56.750791003 +0000 UTC m=+0.017923891 image pull a49113eacc58832dc36e13e21ab00e230e7480fa0f05e368736a82d00d143a48 quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030 2024-04-05T03:16:57.092 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:57 smithi196 podman[106399]: 2024-04-05 03:16:57.088229519 +0000 UTC m=+0.355362394 container init a28bbc73aeed49a2fb6c991d9bad3dd7d3e388af8e3be98bc74c55428d80cfe3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240403, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=quincy-6fda41f, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-05T03:16:57.092 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:57 smithi196 podman[106399]: 2024-04-05 03:16:57.093734874 +0000 UTC m=+0.360867757 container start a28bbc73aeed49a2fb6c991d9bad3dd7d3e388af8e3be98bc74c55428d80cfe3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, RELEASE=quincy-6fda41f, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240403, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-04-05T03:16:57.366 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:57 smithi196 podman[106399]: 2024-04-05 03:16:57.171286838 +0000 UTC m=+0.438419729 container attach a28bbc73aeed49a2fb6c991d9bad3dd7d3e388af8e3be98bc74c55428d80cfe3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-05T03:16:57.635 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:57 smithi196 conmon[106414]: conmon a28bbc73aeed49a2fb6c : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-a28bbc73aeed49a2fb6c991d9bad3dd7d3e388af8e3be98bc74c55428d80cfe3.scope/memory.events 2024-04-05T03:16:57.636 INFO:journalctl@ceph.osd.7.smithi196.stdout:Apr 05 03:16:57 smithi196 podman[106399]: 2024-04-05 03:16:57.369410459 +0000 UTC m=+0.636543334 container died a28bbc73aeed49a2fb6c991d9bad3dd7d3e388af8e3be98bc74c55428d80cfe3 (image=quay.ceph.io/ceph-ci/ceph@sha256:bc7712ec89fede52a2d1211e734d3862c877e03d01eec5cccb6e6be746353030, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-osd-7-deactivate, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240403, RELEASE=quincy-6fda41f, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-05T03:16:57.947 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@osd.7.service' 2024-04-05T03:16:57.990 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:57.990 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-05T03:16:57.990 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-05T03:16:57.991 DEBUG:teuthology.orchestra.run.smithi196:> sudo systemctl stop ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a 2024-04-05T03:16:58.329 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 systemd[1]: Stopping Ceph prometheus.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:16:58.636 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:16:58.636 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-05T03:16:58.636 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-05T03:16:58.636 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-05T03:16:58.636 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.330Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.331Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.331Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a[80518]: ts=2024-04-05T03:16:58.331Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-05T03:16:58.637 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 podman[106528]: 2024-04-05 03:16:58.340696132 +0000 UTC m=+0.119292745 container died dc41e7016befea0e504a0f2ebe26d3681427fb329801eeca3f819a2ca3885c47 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:16:59.059 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 podman[106528]: 2024-04-05 03:16:58.851891463 +0000 UTC m=+0.630488082 container cleanup dc41e7016befea0e504a0f2ebe26d3681427fb329801eeca3f819a2ca3885c47 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-05T03:16:59.060 INFO:journalctl@ceph.prometheus.a.smithi196.stdout:Apr 05 03:16:58 smithi196 bash[106528]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-prometheus-a 2024-04-05T03:16:59.265 DEBUG:teuthology.orchestra.run.smithi196:> sudo pkill -f 'journalctl -f -n 0 -u ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@prometheus.a.service' 2024-04-05T03:16:59.301 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-05T03:16:59.301 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-05T03:16:59.301 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-05T03:17:00.419 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:17:00.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:17:00.419 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:17:00.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi150.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.150:8443: connect: connection refused" 2024-04-05T03:17:00.419 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:00 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:17:00.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi196.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.196:8443: connect: connection refused" 2024-04-05T03:17:01.348 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:01 smithi150 systemd[1]: Stopping Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:17:01.653 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:01 smithi150 ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a[91614]: ts=2024-04-05T03:17:01.350Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-05T03:17:01.653 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:01 smithi150 podman[167102]: 2024-04-05 03:17:01.391383036 +0000 UTC m=+0.145030966 container died 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:17:01.976 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:01 smithi150 podman[167102]: 2024-04-05 03:17:01.839440485 +0000 UTC m=+0.593088411 container cleanup 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:17:01.977 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:01 smithi150 bash[167102]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a 2024-04-05T03:17:02.390 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:02 smithi150 podman[167121]: 2024-04-05 03:17:02.11565272 +0000 UTC m=+0.722127300 container remove 5c82bec44fb970234e424e6cb6d274b9c0edd1d4755371aafa08749af2431b9d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-05T03:17:02.391 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:02 smithi150 podman[167121]: 2024-04-05 03:17:02.191291044 +0000 UTC m=+0.797765627 volume remove ef885945dca93dbaddb4508256098671422b25ca504f798def71d2d21e72fc7a 2024-04-05T03:17:02.726 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:02 smithi150 systemd[1]: Stopping Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:17:02.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:02 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@alertmanager.a.service: Deactivated successfully. 2024-04-05T03:17:02.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:02 smithi150 systemd[1]: Stopped Ceph alertmanager.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:17:02.727 INFO:journalctl@ceph.alertmanager.a.smithi150.stdout:Apr 05 03:17:02 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@alertmanager.a.service: Consumed 2.814s CPU time. 2024-04-05T03:17:03.092 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:02 smithi150 podman[167232]: 2024-04-05 03:17:02.832240504 +0000 UTC m=+0.110309762 container died dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:17:03.451 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 podman[167232]: 2024-04-05 03:17:03.209388778 +0000 UTC m=+0.487458035 container cleanup dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:17:03.451 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 bash[167232]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a 2024-04-05T03:17:03.726 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 podman[167249]: 2024-04-05 03:17:03.453110415 +0000 UTC m=+0.622081108 container remove dc5e73ff394f7aaf45e1abe3f1b66a86e34bed01ffdb845739df44b82aaef3d2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-05T03:17:03.726 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:17:03.727 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-05T03:17:03.727 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 systemd[1]: Stopped Ceph node-exporter.a for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:17:03.727 INFO:journalctl@ceph.node-exporter.a.smithi150.stdout:Apr 05 03:17:03 smithi150 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.a.service: Consumed 4.170s CPU time. 2024-04-05T03:17:11.750 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 --force --keep-logs 2024-04-05T03:17:13.633 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:13 smithi196 systemd[1]: Stopping Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4... 2024-04-05T03:17:13.885 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:13 smithi196 podman[106984]: 2024-04-05 03:17:13.63530134 +0000 UTC m=+0.128249328 container died 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:17:14.147 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:13 smithi196 podman[106984]: 2024-04-05 03:17:13.971329479 +0000 UTC m=+0.464277463 container cleanup 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:17:14.148 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:13 smithi196 bash[106984]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b 2024-04-05T03:17:14.148 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:14 smithi196 podman[107002]: 2024-04-05 03:17:14.147552351 +0000 UTC m=+0.514454765 container remove 0d264938c335dcb730d63dae056055a80463da4902bcaa43a5490a079da08fe2 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-05T03:17:14.555 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:14 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-05T03:17:14.555 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:14 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-05T03:17:14.555 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:14 smithi196 systemd[1]: Stopped Ceph node-exporter.b for 15e4009a-f2f6-11ee-b649-cb9ed24678a4. 2024-04-05T03:17:14.555 INFO:journalctl@ceph.node-exporter.b.smithi196.stdout:Apr 05 03:17:14 smithi196 systemd[1]: ceph-15e4009a-f2f6-11ee-b649-cb9ed24678a4@node-exporter.b.service: Consumed 3.857s CPU time. 2024-04-05T03:17:18.315 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:17:18.351 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-05T03:17:18.381 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-05T03:17:18.383 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/remote/smithi150/crash 2024-04-05T03:17:18.384 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/crash -- . 2024-04-05T03:17:18.443 INFO:teuthology.orchestra.run.smithi150.stderr:tar: /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-05T03:17:18.443 INFO:teuthology.orchestra.run.smithi150.stderr:tar: Error is not recoverable: exiting now 2024-04-05T03:17:18.445 DEBUG:teuthology.misc:Transferring archived files from smithi196:/var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/crash to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/remote/smithi196/crash 2024-04-05T03:17:18.446 DEBUG:teuthology.orchestra.run.smithi196:> sudo tar c -f - -C /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/crash -- . 2024-04-05T03:17:18.487 INFO:teuthology.orchestra.run.smithi196.stderr:tar: /var/lib/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/crash: Cannot open: No such file or directory 2024-04-05T03:17:18.487 INFO:teuthology.orchestra.run.smithi196.stderr:tar: Error is not recoverable: exiting now 2024-04-05T03:17:18.488 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-05T03:17:18.488 DEBUG:teuthology.orchestra.run.smithi150:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-05T03:17:18.538 INFO:teuthology.orchestra.run.smithi150.stdout:1712285161.1371624 mon.a (mon.0) 374 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-05T03:17:18.538 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-05T03:17:18.538 INFO:tasks.cephadm:Compressing logs... 2024-04-05T03:17:18.538 DEBUG:teuthology.orchestra.run.smithi150:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-05T03:17:18.541 DEBUG:teuthology.orchestra.run.smithi196:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-05T03:17:18.567 INFO:teuthology.orchestra.run.smithi196.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-05T03:17:18.568 INFO:teuthology.orchestra.run.smithi196.stderr:: No such file or directory 2024-04-05T03:17:18.568 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.6.log 2024-04-05T03:17:18.568 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.audit.log 2024-04-05T03:17:18.568 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.4.log 2024-04-05T03:17:18.568 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.6.log: /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.7.log 2024-04-05T03:17:18.569 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.5.log 2024-04-05T03:17:18.570 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.4.log: /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mgr.x.log 2024-04-05T03:17:18.570 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi196.eohomm.log 2024-04-05T03:17:18.571 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.cephadm.log 2024-04-05T03:17:18.572 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi196.eohomm.log: 62.6% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi196.eohomm.log.gz 2024-04-05T03:17:18.572 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log 2024-04-05T03:17:18.572 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.cephadm.log: 82.7% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-05T03:17:18.572 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-volume.log 2024-04-05T03:17:18.573 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log: 89.4% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.audit.log.gz 2024-04-05T03:17:18.573 INFO:teuthology.orchestra.run.smithi196.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.b.log 2024-04-05T03:17:18.577 INFO:teuthology.orchestra.run.smithi196.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-volume.log: /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.b.log: 87.6% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log.gz 2024-04-05T03:17:18.589 INFO:teuthology.orchestra.run.smithi196.stderr: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-05T03:17:18.596 INFO:teuthology.orchestra.run.smithi150.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-04-05T03:17:18.596 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-05T03:17:18.597 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.a.log 2024-04-05T03:17:18.598 INFO:teuthology.orchestra.run.smithi150.stderr:gzip -5 --verbose --/var/log/ceph/cephadm.log: /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi150.jnrwgs.log 2024-04-05T03:17:18.598 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.audit.log 2024-04-05T03:17:18.598 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi150.jnrwgs.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.1.log 2024-04-05T03:17:18.598 INFO:teuthology.orchestra.run.smithi150.stderr: 82.1% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mds.foo.smithi150.jnrwgs.log.gz 2024-04-05T03:17:18.598 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.3.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mgr.y.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.0.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.cephadm.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.2.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-volume.log 2024-04-05T03:17:18.601 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.c.log 2024-04-05T03:17:18.605 INFO:teuthology.orchestra.run.smithi150.stderr:/var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-volume.log: /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.c.log: 92.8% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.log.gz 2024-04-05T03:17:18.628 INFO:teuthology.orchestra.run.smithi196.stderr: 91.2% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-volume.log.gz 2024-04-05T03:17:18.632 INFO:teuthology.orchestra.run.smithi196.stderr: 90.1% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mgr.x.log.gz 2024-04-05T03:17:18.667 INFO:teuthology.orchestra.run.smithi150.stderr: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-05T03:17:18.783 INFO:teuthology.orchestra.run.smithi150.stderr: 90.0% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.cephadm.log.gz 2024-04-05T03:17:18.835 INFO:teuthology.orchestra.run.smithi150.stderr: 91.0% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-volume.log.gz 2024-04-05T03:17:18.999 INFO:teuthology.orchestra.run.smithi196.stderr: 92.0% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.b.log.gz 2024-04-05T03:17:19.016 INFO:teuthology.orchestra.run.smithi150.stderr: 93.6% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph.audit.log.gz 2024-04-05T03:17:19.680 INFO:teuthology.orchestra.run.smithi150.stderr: 88.8% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mgr.y.log.gz 2024-04-05T03:17:20.957 INFO:teuthology.orchestra.run.smithi150.stderr: 91.6% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.c.log.gz 2024-04-05T03:17:22.358 INFO:teuthology.orchestra.run.smithi150.stderr: 90.4% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-mon.a.log.gz 2024-04-05T03:17:57.488 INFO:teuthology.orchestra.run.smithi196.stderr: 93.6% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.6.log.gz 2024-04-05T03:17:58.534 INFO:teuthology.orchestra.run.smithi196.stderr: 93.7% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.7.log.gz 2024-04-05T03:17:59.010 INFO:teuthology.orchestra.run.smithi196.stderr: 93.6% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.4.log.gz 2024-04-05T03:18:14.422 INFO:teuthology.orchestra.run.smithi196.stderr: 93.3% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.5.log.gz 2024-04-05T03:18:14.424 INFO:teuthology.orchestra.run.smithi196.stderr: 2024-04-05T03:18:14.424 INFO:teuthology.orchestra.run.smithi196.stderr:real 0m55.869s 2024-04-05T03:18:14.424 INFO:teuthology.orchestra.run.smithi196.stderr:user 2m48.845s 2024-04-05T03:18:14.424 INFO:teuthology.orchestra.run.smithi196.stderr:sys 0m6.466s 2024-04-05T03:19:00.567 INFO:teuthology.orchestra.run.smithi150.stderr: 93.4% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.0.log.gz 2024-04-05T03:19:25.469 INFO:teuthology.orchestra.run.smithi150.stderr: 93.0% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.3.log.gz 2024-04-05T03:20:00.651 INFO:teuthology.orchestra.run.smithi150.stderr: 93.5% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.1.log.gz 2024-04-05T03:20:15.441 INFO:teuthology.orchestra.run.smithi150.stderr: 93.3% -- replaced with /var/log/ceph/15e4009a-f2f6-11ee-b649-cb9ed24678a4/ceph-osd.2.log.gz 2024-04-05T03:20:15.443 INFO:teuthology.orchestra.run.smithi150.stderr: 2024-04-05T03:20:15.443 INFO:teuthology.orchestra.run.smithi150.stderr:real 2m56.890s 2024-04-05T03:20:15.443 INFO:teuthology.orchestra.run.smithi150.stderr:user 3m49.569s 2024-04-05T03:20:15.443 INFO:teuthology.orchestra.run.smithi150.stderr:sys 0m9.448s 2024-04-05T03:20:15.444 INFO:tasks.cephadm:Archiving logs... 2024-04-05T03:20:15.444 DEBUG:teuthology.misc:Transferring archived files from smithi150:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/remote/smithi150/log 2024-04-05T03:20:15.445 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-05T03:20:51.783 DEBUG:teuthology.misc:Transferring archived files from smithi196:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/remote/smithi196/log 2024-04-05T03:20:51.785 DEBUG:teuthology.orchestra.run.smithi196:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-05T03:21:18.450 INFO:tasks.cephadm:Removing cluster... 2024-04-05T03:21:18.450 DEBUG:teuthology.orchestra.run.smithi150:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 --force 2024-04-05T03:21:19.229 DEBUG:teuthology.orchestra.run.smithi196:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 15e4009a-f2f6-11ee-b649-cb9ed24678a4 --force 2024-04-05T03:21:19.979 INFO:tasks.cephadm:Removing cephadm ... 2024-04-05T03:21:19.979 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-05T03:21:19.997 DEBUG:teuthology.orchestra.run.smithi196:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-05T03:21:20.012 INFO:tasks.cephadm:Teardown complete 2024-04-05T03:21:20.012 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-05T03:21:20.023 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-05T03:21:20.023 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-05T03:21:20.040 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-05T03:21:20.139 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-05T03:21:20.148 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-05T03:21:20.148 DEBUG:teuthology.orchestra.run.smithi150:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T03:21:20.151 DEBUG:teuthology.orchestra.run.smithi196:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-05T03:21:20.164 INFO:teuthology.orchestra.run.smithi150.stderr:bash: line 1: ntpq: command not found 2024-04-05T03:21:20.165 INFO:teuthology.orchestra.run.smithi196.stderr:bash: line 1: ntpq: command not found 2024-04-05T03:21:20.168 INFO:teuthology.orchestra.run.smithi196.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T03:21:20.169 INFO:teuthology.orchestra.run.smithi196.stdout:=============================================================================== 2024-04-05T03:21:20.169 INFO:teuthology.orchestra.run.smithi196.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 52 -1271us[-1271us] +/- 82ms 2024-04-05T03:21:20.169 INFO:teuthology.orchestra.run.smithi196.stdout:^* hv02.front.sepia.ceph.com 3 6 377 53 +255us[ +268us] +/- 70ms 2024-04-05T03:21:20.169 INFO:teuthology.orchestra.run.smithi196.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 54 +811us[ +824us] +/- 71ms 2024-04-05T03:21:20.169 INFO:teuthology.orchestra.run.smithi196.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T03:21:20.178 INFO:teuthology.orchestra.run.smithi150.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-05T03:21:20.178 INFO:teuthology.orchestra.run.smithi150.stdout:=============================================================================== 2024-04-05T03:21:20.179 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 56 -1279us[-1297us] +/- 82ms 2024-04-05T03:21:20.179 INFO:teuthology.orchestra.run.smithi150.stdout:^* hv02.front.sepia.ceph.com 3 6 377 55 +238us[ +219us] +/- 70ms 2024-04-05T03:21:20.179 INFO:teuthology.orchestra.run.smithi150.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 +830us[ +812us] +/- 71ms 2024-04-05T03:21:20.179 INFO:teuthology.orchestra.run.smithi150.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-05T03:21:20.180 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-05T03:21:20.190 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-05T03:21:20.190 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-05T03:21:20.228 DEBUG:teuthology.orchestra.run.smithi150:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-05T03:21:20.388 DEBUG:teuthology.orchestra.run.smithi196:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-05T03:21:20.469 DEBUG:teuthology.orchestra.run.smithi150:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T03:21:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:21:20.502 DEBUG:teuthology.orchestra.run.smithi196:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-04-05T03:21:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:21:20.533 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-05T03:21:20.543 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-05T03:21:20.570 INFO:teuthology.task.internal:Duration was 2773.799157 seconds 2024-04-05T03:21:20.570 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-05T03:21:20.605 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-05T03:21:20.605 DEBUG:teuthology.orchestra.run.smithi150:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-05T03:21:20.608 DEBUG:teuthology.orchestra.run.smithi196:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-05T03:21:20.650 INFO:teuthology.orchestra.run.smithi196.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T03:21:20.687 INFO:teuthology.orchestra.run.smithi150.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-05T03:21:21.028 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-05T03:21:21.028 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi150.front.sepia.ceph.com 2024-04-05T03:21:21.029 DEBUG:teuthology.orchestra.run.smithi150:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-05T03:21:21.048 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi196.front.sepia.ceph.com 2024-04-05T03:21:21.048 DEBUG:teuthology.orchestra.run.smithi196:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-05T03:21:21.081 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-05T03:21:21.081 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-05T03:21:21.091 DEBUG:teuthology.orchestra.run.smithi196:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-05T03:21:21.150 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-05T03:21:21.150 DEBUG:teuthology.orchestra.run.smithi150:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-05T03:21:21.159 DEBUG:teuthology.orchestra.run.smithi196:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-05T03:21:21.627 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-05T03:21:21.640 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-05T03:21:21.641 DEBUG:teuthology.orchestra.run.smithi150:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-05T03:21:21.691 DEBUG:teuthology.orchestra.run.smithi196:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-05T03:21:21.718 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-05T03:21:21.728 DEBUG:teuthology.orchestra.run.smithi150:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:21:21.730 DEBUG:teuthology.orchestra.run.smithi196:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:21:21.758 INFO:teuthology.orchestra.run.smithi150.stdout:kernel.core_pattern = core 2024-04-05T03:21:21.783 INFO:teuthology.orchestra.run.smithi196.stdout:kernel.core_pattern = core 2024-04-05T03:21:21.808 DEBUG:teuthology.orchestra.run.smithi150:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:21:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:21:21.832 DEBUG:teuthology.orchestra.run.smithi196:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-05T03:21:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-05T03:21:21.862 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-05T03:21:21.873 INFO:teuthology.task.internal:Transferring archived files... 2024-04-05T03:21:21.874 DEBUG:teuthology.misc:Transferring archived files from smithi150:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/remote/smithi150 2024-04-05T03:21:21.874 DEBUG:teuthology.orchestra.run.smithi150:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-05T03:21:21.926 DEBUG:teuthology.misc:Transferring archived files from smithi196:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230/remote/smithi196 2024-04-05T03:21:21.926 DEBUG:teuthology.orchestra.run.smithi196:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-05T03:21:21.966 INFO:teuthology.task.internal:Removing archive directory... 2024-04-05T03:21:21.966 DEBUG:teuthology.orchestra.run.smithi150:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-05T03:21:21.969 DEBUG:teuthology.orchestra.run.smithi196:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-05T03:21:22.014 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-05T03:21:22.026 INFO:teuthology.task.internal:Not uploading archives. 2024-04-05T03:21:22.026 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-05T03:21:22.054 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-05T03:21:22.054 DEBUG:teuthology.orchestra.run.smithi150:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-05T03:21:22.057 DEBUG:teuthology.orchestra.run.smithi196:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-05T03:21:22.069 INFO:teuthology.orchestra.run.smithi150.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 5 03:21 /home/ubuntu/cephtest 2024-04-05T03:21:22.071 INFO:teuthology.orchestra.run.smithi196.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 5 03:21 /home/ubuntu/cephtest 2024-04-05T03:21:22.072 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-05T03:21:22.087 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-05T03:21:22.154 INFO:teuthology.nuke:Checking targets against current locks 2024-04-05T03:21:22.182 DEBUG:teuthology.nuke:shortname: smithi150 2024-04-05T03:21:22.183 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-05T03:21:22.204 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi150.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:25:58.551901', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMIuUwNr2r/Tbm93vuIaJO2JQe+gvaskSlRuWjiq8XpYbdkqX6eS/NfxknSXO39TxGsSTrzk5nRpajplQ0nP90Y='} 2024-04-05T03:21:22.210 DEBUG:teuthology.nuke:shortname: smithi196 2024-04-05T03:21:22.211 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-05T03:21:22.231 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi196.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-05_02:08:13-upgrade-squid-distro-default-smithi/7641230', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-05 02:25:58.549985', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO3Or61tPDasmByHPKt6guIqEGtWzShtMvTuJPVqqDYiej8lBCagYKGCJwl0jbWUkfeX9SVuJTH+vH9/kpATp+w='} 2024-04-05T03:21:22.258 INFO:teuthology.orchestra.console.smithi150:Power off 2024-04-05T03:21:22.259 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-05T03:21:22.282 DEBUG:teuthology.orchestra.console.smithi150:power off output: Chassis Power Control: Down/Off 2024-04-05T03:21:22.282 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:22.300 INFO:teuthology.orchestra.console.smithi196:Power off 2024-04-05T03:21:22.300 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-05T03:21:22.310 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2024-04-05T03:21:22.323 DEBUG:teuthology.orchestra.console.smithi196:power off output: Chassis Power Control: Down/Off 2024-04-05T03:21:22.323 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:22.346 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is on 2024-04-05T03:21:26.311 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:26.347 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:26.423 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is on 2024-04-05T03:21:26.458 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is on 2024-04-05T03:21:30.426 DEBUG:teuthology.orchestra.console.smithi150:pexpect command: ipmitool -H smithi150.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:30.459 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:30.537 DEBUG:teuthology.orchestra.console.smithi150:check power output: Chassis Power is off 2024-04-05T03:21:30.569 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is on 2024-04-05T03:21:30.638 INFO:teuthology.orchestra.console.smithi150:Power off completed 2024-04-05T03:21:34.573 DEBUG:teuthology.orchestra.console.smithi196:pexpect command: ipmitool -H smithi196.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-05T03:21:34.684 DEBUG:teuthology.orchestra.console.smithi196:check power output: Chassis Power is off 2024-04-05T03:21:34.784 INFO:teuthology.orchestra.console.smithi196:Power off completed 2024-04-05T03:21:34.889 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 2773.7991569042206 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=35045d7b80ab47549d842577f8e5c177 status: fail success: false 2024-04-05T03:21:34.890 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-05T03:21:34.972 INFO:teuthology.run:FAIL